From 88fdeb5085a2de4448c5e874900a8846dbc5bdaf Mon Sep 17 00:00:00 2001 From: Christian Lachner Date: Fri, 16 Mar 2018 11:26:28 +0100 Subject: [PATCH] haproxy: Update MEDIUM+ patches for HAProxy v1.8.4 - Add new MEDIUM+ patches (see https://www.haproxy.org/bugs/bugs-1.8.4.html) - Raise patch-level to 02 Signed-off-by: Christian Lachner --- net/haproxy/Makefile | 2 +- ...ing-data-after-end-of-output-buffers.patch | 71 +++++++++++++++++++ ...r-Fix-the-wrapping-case-in-bo_putblk.patch | 33 +++++++++ ...r-Fix-the-wrapping-case-in-bi_putblk.patch | 33 +++++++++ ...also-arm-the-h2-timeout-when-sending.patch | 46 ++++++++++++ ...age-with-cpu-map-and-nbthread-nbproc.patch | 55 ++++++++++++++ ...m-idle-list-when-HAProxy-is-stopping.patch | 43 +++++++++++ 7 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 net/haproxy/patches/0005-BUG-MEDIUM-h2-always-consume-any-trailing-data-after-end-of-output-buffers.patch create mode 100644 net/haproxy/patches/0006-BUG-MEDIUM-buffer-Fix-the-wrapping-case-in-bo_putblk.patch create mode 100644 net/haproxy/patches/0007-BUG-MEDIUM-buffer-Fix-the-wrapping-case-in-bi_putblk.patch create mode 100644 net/haproxy/patches/0008-BUG-MEDIUM-h2-also-arm-the-h2-timeout-when-sending.patch create mode 100644 net/haproxy/patches/0009-BUG-MEDIUM-fix-a-100-cpu-usage-with-cpu-map-and-nbthread-nbproc.patch create mode 100644 net/haproxy/patches/0010-BUG-MEDIUM-spoe-Remove-idle-applets-from-idle-list-when-HAProxy-is-stopping.patch diff --git a/net/haproxy/Makefile b/net/haproxy/Makefile index da0b3c474..0a37b8f38 100644 --- a/net/haproxy/Makefile +++ b/net/haproxy/Makefile @@ -10,7 +10,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=haproxy PKG_VERSION:=1.8.4 -PKG_RELEASE:=01 +PKG_RELEASE:=02 PKG_SOURCE:=haproxy-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://www.haproxy.org/download/1.8/src/ diff --git a/net/haproxy/patches/0005-BUG-MEDIUM-h2-always-consume-any-trailing-data-after-end-of-output-buffers.patch b/net/haproxy/patches/0005-BUG-MEDIUM-h2-always-consume-any-trailing-data-after-end-of-output-buffers.patch new file mode 100644 index 000000000..2ed041f39 --- /dev/null +++ b/net/haproxy/patches/0005-BUG-MEDIUM-h2-always-consume-any-trailing-data-after-end-of-output-buffers.patch @@ -0,0 +1,71 @@ +From 6fc36785addd45cc76a029a023296def53cff135 Mon Sep 17 00:00:00 2001 +From: Willy Tarreau +Date: Tue, 27 Feb 2018 15:37:25 +0100 +Subject: [PATCH] BUG/MEDIUM: h2: always consume any trailing data after end + of output buffers + +In case a stream tries to emit more data than advertised by the chunks +or content-length headers, the extra data remains in the channel's output +buffer until the channel's timeout expires. It can easily happen when +sending malformed error files making use of a wrong content-length or +having extra CRLFs after the empty chunk. It may also be possible to +forge such a bad response using Lua. + +The H1 to H2 encoder must protect itself against this by marking the data +presented to it as consumed if it decides to discard them, so that the +sending stream doesn't wait for the timeout to trigger. + +The visible effect of this problem is a huge memory usage and a high +concurrent connection count during benchmarks when using such bad data +(a typical place where this easily happens). + +This fix must be backported to 1.8. + +(cherry picked from commit 35a62705df65632e2717ae0d20a93e0cb3f8f163) +Signed-off-by: Willy Tarreau +--- + src/mux_h2.c | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/src/mux_h2.c b/src/mux_h2.c +index caae041..4303a06 100644 +--- a/src/mux_h2.c ++++ b/src/mux_h2.c +@@ -3020,6 +3020,9 @@ static int h2s_frt_make_resp_headers(struct h2s *h2s, struct buffer *buf) + * body or directly end in TRL2. + */ + if (es_now) { ++ // trim any possibly pending data (eg: inconsistent content-length) ++ bo_del(buf, buf->o); ++ + h1m->state = HTTP_MSG_DONE; + h2s->flags |= H2_SF_ES_SENT; + if (h2s->st == H2_SS_OPEN) +@@ -3269,8 +3272,12 @@ static int h2s_frt_make_resp_data(struct h2s *h2s, struct buffer *buf) + else + h2c_stream_close(h2c, h2s); + +- if (!(h1m->flags & H1_MF_CHNK)) ++ if (!(h1m->flags & H1_MF_CHNK)) { ++ // trim any possibly pending data (eg: inconsistent content-length) ++ bo_del(buf, buf->o); ++ + h1m->state = HTTP_MSG_DONE; ++ } + + h2s->flags |= H2_SF_ES_SENT; + } +@@ -3319,6 +3326,10 @@ static int h2_snd_buf(struct conn_stream *cs, struct buffer *buf, int flags) + } + total += count; + bo_del(buf, count); ++ ++ // trim any possibly pending data (eg: extra CR-LF, ...) ++ bo_del(buf, buf->o); ++ + h2s->res.state = HTTP_MSG_DONE; + break; + } +-- +1.7.10.4 + diff --git a/net/haproxy/patches/0006-BUG-MEDIUM-buffer-Fix-the-wrapping-case-in-bo_putblk.patch b/net/haproxy/patches/0006-BUG-MEDIUM-buffer-Fix-the-wrapping-case-in-bo_putblk.patch new file mode 100644 index 000000000..94eec1f6b --- /dev/null +++ b/net/haproxy/patches/0006-BUG-MEDIUM-buffer-Fix-the-wrapping-case-in-bo_putblk.patch @@ -0,0 +1,33 @@ +From fefb8592821ff0fa56f435c581d6e92e563e7ad7 Mon Sep 17 00:00:00 2001 +From: Christopher Faulet +Date: Mon, 26 Feb 2018 10:47:03 +0100 +Subject: [PATCH] BUG/MEDIUM: buffer: Fix the wrapping case in bo_putblk + +When the block of data need to be split to support the wrapping, the start of +the second block of data was wrong. We must be sure to skip data copied during +the first memcpy. + +This patch must be backported to 1.8, 1.7, 1.6 and 1.5. + +(cherry picked from commit b2b279464c5c0f3dfadf02333e06eb0ae8ae8793) +Signed-off-by: Willy Tarreau +--- + include/common/buffer.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/include/common/buffer.h b/include/common/buffer.h +index 976085e..ae9aafd 100644 +--- a/include/common/buffer.h ++++ b/include/common/buffer.h +@@ -468,7 +468,7 @@ static inline int bo_putblk(struct buffer *b, const char *blk, int len) + memcpy(b->p, blk, half); + b->p = b_ptr(b, half); + if (len > half) { +- memcpy(b->p, blk, len - half); ++ memcpy(b->p, blk + half, len - half); + b->p = b_ptr(b, half); + } + b->o += len; +-- +1.7.10.4 + diff --git a/net/haproxy/patches/0007-BUG-MEDIUM-buffer-Fix-the-wrapping-case-in-bi_putblk.patch b/net/haproxy/patches/0007-BUG-MEDIUM-buffer-Fix-the-wrapping-case-in-bi_putblk.patch new file mode 100644 index 000000000..93ca220a9 --- /dev/null +++ b/net/haproxy/patches/0007-BUG-MEDIUM-buffer-Fix-the-wrapping-case-in-bi_putblk.patch @@ -0,0 +1,33 @@ +From 14f325000b91649b9d117c4d53d6b194ed3c7b11 Mon Sep 17 00:00:00 2001 +From: Christopher Faulet +Date: Mon, 26 Feb 2018 10:51:28 +0100 +Subject: [PATCH] BUG/MEDIUM: buffer: Fix the wrapping case in bi_putblk + +When the block of data need to be split to support the wrapping, the start of +the second block of data was wrong. We must be sure to skup data copied during +the first memcpy. + +This patch must be backported to 1.8. + +(cherry picked from commit ca6ef506610e9d78f99b7ab2095ce0f8a47e18df) +Signed-off-by: Willy Tarreau +--- + include/common/buffer.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/include/common/buffer.h b/include/common/buffer.h +index ae9aafd..0e63913 100644 +--- a/include/common/buffer.h ++++ b/include/common/buffer.h +@@ -577,7 +577,7 @@ static inline int bi_putblk(struct buffer *b, const char *blk, int len) + + memcpy(bi_end(b), blk, half); + if (len > half) +- memcpy(b_ptr(b, b->i + half), blk, len - half); ++ memcpy(b_ptr(b, b->i + half), blk + half, len - half); + b->i += len; + return len; + } +-- +1.7.10.4 + diff --git a/net/haproxy/patches/0008-BUG-MEDIUM-h2-also-arm-the-h2-timeout-when-sending.patch b/net/haproxy/patches/0008-BUG-MEDIUM-h2-also-arm-the-h2-timeout-when-sending.patch new file mode 100644 index 000000000..9096eb15a --- /dev/null +++ b/net/haproxy/patches/0008-BUG-MEDIUM-h2-also-arm-the-h2-timeout-when-sending.patch @@ -0,0 +1,46 @@ +From ccfb5d755f1708f890b197375d962d8c938e78bd Mon Sep 17 00:00:00 2001 +From: Willy Tarreau +Date: Mon, 5 Mar 2018 16:10:54 +0100 +Subject: [PATCH] BUG/MEDIUM: h2: also arm the h2 timeout when sending + +Right now the h2 idle timeout is only set when there is no stream. If we +fail to send because the socket buffers are full (generally indicating +the client has left), we also need to arm it so that we can properly +expire such connections, otherwise some failed transfers might leave +H2 connections pending forever. + +Thanks to Thierry Fournier for the diag and the traces. + +This patch needs to be backported to 1.8. + +(cherry picked from commit 84b118f3120b3c61156f0ada12ae6456bd1a0b5a) +Signed-off-by: Willy Tarreau +--- + src/mux_h2.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/mux_h2.c b/src/mux_h2.c +index 4303a06..5446fd4 100644 +--- a/src/mux_h2.c ++++ b/src/mux_h2.c +@@ -2329,7 +2329,7 @@ static int h2_wake(struct connection *conn) + } + + if (h2c->task) { +- if (eb_is_empty(&h2c->streams_by_id)) { ++ if (eb_is_empty(&h2c->streams_by_id) || h2c->mbuf->o) { + h2c->task->expire = tick_add(now_ms, h2c->last_sid < 0 ? h2c->timeout : h2c->shut_timeout); + task_queue(h2c->task); + } +@@ -2501,7 +2501,7 @@ static void h2_detach(struct conn_stream *cs) + h2_release(h2c->conn); + } + else if (h2c->task) { +- if (eb_is_empty(&h2c->streams_by_id)) { ++ if (eb_is_empty(&h2c->streams_by_id) || h2c->mbuf->o) { + h2c->task->expire = tick_add(now_ms, h2c->last_sid < 0 ? h2c->timeout : h2c->shut_timeout); + task_queue(h2c->task); + } +-- +1.7.10.4 + diff --git a/net/haproxy/patches/0009-BUG-MEDIUM-fix-a-100-cpu-usage-with-cpu-map-and-nbthread-nbproc.patch b/net/haproxy/patches/0009-BUG-MEDIUM-fix-a-100-cpu-usage-with-cpu-map-and-nbthread-nbproc.patch new file mode 100644 index 000000000..23a4028ca --- /dev/null +++ b/net/haproxy/patches/0009-BUG-MEDIUM-fix-a-100-cpu-usage-with-cpu-map-and-nbthread-nbproc.patch @@ -0,0 +1,55 @@ +From 5149cd3c7abad68ddb19a0a5b3b604786d5f1b95 Mon Sep 17 00:00:00 2001 +From: =?utf8?q?Cyril=20Bont=C3=A9?= +Date: Mon, 12 Mar 2018 21:47:39 +0100 +Subject: [PATCH] BUG/MEDIUM: fix a 100% cpu usage with cpu-map and + nbthread/nbproc + +Krishna Kumar reported a 100% cpu usage with a configuration using +cpu-map and a high number of threads, + +Indeed, this minimal configuration to reproduce the issue : + global + nbthread 40 + cpu-map auto:1/1-40 0-39 + + frontend test + bind :8000 + +This is due to a wrong type in a shift operator (int vs unsigned long int), +causing an endless loop while applying the cpu affinity on threads. The same +issue may also occur with nbproc under FreeBSD. This commit addresses both +cases. + +This patch must be backported to 1.8. + +(cherry picked from commit d400ab3a369523538c426cb70e059954c76b69c3) +Signed-off-by: Willy Tarreau +--- + src/haproxy.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/haproxy.c b/src/haproxy.c +index 09f7b5e..7d6e019 100644 +--- a/src/haproxy.c ++++ b/src/haproxy.c +@@ -2838,7 +2838,7 @@ int main(int argc, char **argv) + CPU_ZERO(&cpuset); + while ((i = ffsl(cpu_map)) > 0) { + CPU_SET(i - 1, &cpuset); +- cpu_map &= ~(1 << (i - 1)); ++ cpu_map &= ~(1UL << (i - 1)); + } + ret = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cpuset), &cpuset); + } +@@ -3038,7 +3038,7 @@ int main(int argc, char **argv) + + while ((j = ffsl(cpu_map)) > 0) { + CPU_SET(j - 1, &cpuset); +- cpu_map &= ~(1 << (j - 1)); ++ cpu_map &= ~(1UL << (j - 1)); + } + pthread_setaffinity_np(threads[i], + sizeof(cpuset), &cpuset); +-- +1.7.10.4 + diff --git a/net/haproxy/patches/0010-BUG-MEDIUM-spoe-Remove-idle-applets-from-idle-list-when-HAProxy-is-stopping.patch b/net/haproxy/patches/0010-BUG-MEDIUM-spoe-Remove-idle-applets-from-idle-list-when-HAProxy-is-stopping.patch new file mode 100644 index 000000000..ca714224b --- /dev/null +++ b/net/haproxy/patches/0010-BUG-MEDIUM-spoe-Remove-idle-applets-from-idle-list-when-HAProxy-is-stopping.patch @@ -0,0 +1,43 @@ +From 7034083b5063d28276b986d645d18071aba5f4d5 Mon Sep 17 00:00:00 2001 +From: Christopher Faulet +Date: Wed, 28 Feb 2018 13:33:26 +0100 +Subject: [PATCH] BUG/MEDIUM: spoe: Remove idle applets from idle list when + HAProxy is stopping + +In the SPOE applet's handler, when an applet is switched from the state IDLE to +PROCESSING, it is removed for the list of idle applets. But when HAProxy is +stopping, this applet can be switched to DISCONNECT. In this case, we also need +to remove it from the list of idle applets. Else the applet is removed but still +present in the list. It could lead to a segmentation fault or an infinite loop, +depending the code path. + +(cherry picked from commit 7d9f1ba246055046eed547fa35aa546683021dce) +[wt: adapted context for 1.8] +Signed-off-by: Willy Tarreau +--- + src/flt_spoe.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/flt_spoe.c b/src/flt_spoe.c +index 8fb6e0b..e76a352 100644 +--- a/src/flt_spoe.c ++++ b/src/flt_spoe.c +@@ -1866,6 +1866,7 @@ spoe_handle_appctx(struct appctx *appctx) + goto switchstate; + + case SPOE_APPCTX_ST_IDLE: ++ agent->rt[tid].applets_idle--; + if (stopping && + LIST_ISEMPTY(&agent->rt[tid].sending_queue) && + LIST_ISEMPTY(&SPOE_APPCTX(appctx)->waiting_queue)) { +@@ -1874,7 +1875,6 @@ spoe_handle_appctx(struct appctx *appctx) + appctx->st0 = SPOE_APPCTX_ST_DISCONNECT; + goto switchstate; + } +- agent->rt[tid].applets_idle--; + appctx->st0 = SPOE_APPCTX_ST_PROCESSING; + /* fall through */ + +-- +1.7.10.4 +