From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:11 +0000 (UTC) Subject: [master] 8d0cb3f5c vtc_http2: Print headers as "name: value" Message-ID: <20240404142911.82FE7101781@lists.varnish-cache.org> commit 8d0cb3f5c54a2900de3311f340d6024b3cc7aa5e Author: Dridi Boukelmoune Date: Wed Mar 27 11:54:43 2024 +0100 vtc_http2: Print headers as "name: value" The extra space before the colon looked uncanny. The rest is just code indentation improvements. Better diff with the --ignore-all-space --word-diff options. diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 93c7d6508..1d401ed17 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -467,22 +467,20 @@ decode_hdr(struct http *hp, struct hpk_hdr *h, const struct vsb *vsb) r = HPK_DecHdr(iter, h + n); if (r == hpk_err ) break; - vtc_log(hp->vl, 4, - "header[%2d]: %s : %s", - n, - h[n].key.ptr, - h[n].value.ptr); + vtc_log(hp->vl, 4, "header[%2d]: %s: %s", + n, h[n].key.ptr, h[n].value.ptr); n++; if (r == hpk_done) break; } - if (r != hpk_done) + if (r != hpk_done) { vtc_log(hp->vl, hp->fatal ? 4 : 0, - "Header decoding failed (%d) %d", r, hp->fatal); - else if (n == MAX_HDR) + "Header decoding failed (%d) %d", r, hp->fatal); + } else if (n == MAX_HDR) { vtc_log(hp->vl, hp->fatal, - "Max number of headers reached (%d)", MAX_HDR); + "Max number of headers reached (%d)", MAX_HDR); + } HPK_FreeIter(iter); } From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:11 +0000 (UTC) Subject: [master] 7c684d90f txt: New macros to work with strings Message-ID: <20240404142911.96940101784@lists.varnish-cache.org> commit 7c684d90f21eda97afeeeb9712044446ca4696b7 Author: Dridi Boukelmoune Date: Wed Mar 27 15:19:01 2024 +0100 txt: New macros to work with strings diff --git a/include/vdef.h b/include/vdef.h index 2df601119..a321407d7 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -263,6 +263,8 @@ typedef struct { #define Tcheck(t) do { (void)pdiff((t).b, (t).e); } while (0) #define Tlen(t) (pdiff((t).b, (t).e)) +#define Tstr(s) ((txt){(s), (s) + strlen(s)}) +#define Tstrcmp(t, s) (strncmp((t).b, (s), Tlen(t))) /* #3020 dummy definitions until PR is merged*/ #define LIKELY(x) (x) From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:11 +0000 (UTC) Subject: [master] b7bc7f569 http2_hpack: Reorganize header addition for clarity Message-ID: <20240404142911.B04F0101787@lists.varnish-cache.org> commit b7bc7f569c3aa42499a5b9da5e1977d6fd43e9a6 Author: Dridi Boukelmoune Date: Wed Mar 27 16:09:19 2024 +0100 http2_hpack: Reorganize header addition for clarity Instead of passing both a decoder and individual decoder fields, the signature for h2h_addhdr() changed to only take the decoder. The order of parameters is destination first, then the source following the calling conventon of functions like memcpy(). Internally the function is reorganized with a bunch of txt variables to keep track of the header being added, its name and value. In addition to clarity, this also helps improve safety and correctness. For example the :authority pseudo-header name is erased in place to turn it into a regular host header, but having a dedicated txt for the header name allows its preservation. diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index daf4b93c5..48e6e5c5c 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -39,6 +39,18 @@ #include "http2/cache_http2.h" #include "vct.h" +static void +h2h_assert_ready(struct h2h_decode *d) +{ + + CHECK_OBJ_NOTNULL(d, H2H_DECODE_MAGIC); + AN(d->out); + assert(d->namelen >= 2); /* 2 chars from the ": " that we added */ + assert(d->namelen <= d->out_u); + assert(d->out[d->namelen - 2] == ':'); + assert(d->out[d->namelen - 1] == ' '); +} + // rfc9113,l,2493,2528 static h2_error h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) @@ -127,132 +139,130 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } static h2_error -h2h_addhdr(struct h2h_decode *d, struct http *hp, char *b, size_t namelen, - size_t len) +h2h_addhdr(struct http *hp, struct h2h_decode *d) { /* XXX: This might belong in cache/cache_http.c */ - const char *b0; + txt hdr, nm, val; int disallow_empty; + const char *p; unsigned n; - char *p; - unsigned u; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - AN(b); - assert(namelen >= 2); /* 2 chars from the ': ' that we added */ - assert(namelen <= len); + h2h_assert_ready(d); + + /* Assume hdr is by default a regular header from what we decoded. */ + hdr.b = d->out; + hdr.e = hdr.b + d->out_u; + n = hp->nhd; + + /* nm and val are separated by ": " */ + nm.b = hdr.b; + nm.e = nm.b + d->namelen - 2; + val.b = nm.e + 2; + val.e = hdr.e; disallow_empty = 0; - if (len > UINT_MAX) { /* XXX: cache_param max header size */ - VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", b); + if (Tlen(hdr) > UINT_MAX) { /* XXX: cache_param max header size */ + VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", hdr.b); return (H2SE_ENHANCE_YOUR_CALM); } - b0 = b; - if (b[0] == ':') { + if (*nm.b == ':') { /* Match H/2 pseudo headers */ /* XXX: Should probably have some include tbl for pseudo-headers */ - if (!strncmp(b, ":method: ", namelen)) { - b += namelen; - len -= namelen; + if (!Tstrcmp(nm, ":method")) { + hdr.b = val.b; n = HTTP_HDR_METHOD; disallow_empty = 1; - /* First field cannot contain SP or CTL */ - for (p = b, u = 0; u < len; p++, u++) { - if (vct_issp(*p) || vct_isctl(*p)) + /* Check HTTP token */ + for (p = hdr.b; p < hdr.e; p++) { + if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); } - } else if (!strncmp(b, ":path: ", namelen)) { - b += namelen; - len -= namelen; + } else if (!Tstrcmp(nm, ":path")) { + hdr.b = val.b; n = HTTP_HDR_URL; disallow_empty = 1; // rfc9113,l,2693,2705 - if (len > 0 && *b != '/' && - strncmp(b, "*", len) != 0) { + if (Tlen(val) > 0 && *val.b != '/' && + Tstrcmp(val, "*")) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal :path pseudo-header %.*s", - (int)len, b); + (int)Tlen(val), val.b); return (H2SE_PROTOCOL_ERROR); } - /* Second field cannot contain LWS or CTL */ - for (p = b, u = 0; u < len; p++, u++) { + /* Path cannot contain LWS or CTL */ + for (p = hdr.b; p < hdr.e; p++) { if (vct_islws(*p) || vct_isctl(*p)) return (H2SE_PROTOCOL_ERROR); } - } else if (!strncmp(b, ":scheme: ", namelen)) { + } else if (!Tstrcmp(nm, ":scheme")) { /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ if (d->has_scheme) { VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header %.*s%.*s", - (int)namelen, b0, - (int)(len > 20 ? 20 : len), b); + "Duplicate pseudo-header :scheme: %.*s", + vmin_t(int, Tlen(val), 20), val.b); return (H2SE_PROTOCOL_ERROR); } - b++; - len-=1; - n = hp->nhd; + hdr.b++; d->has_scheme = 1; + disallow_empty = 1; - for (p = b + namelen, u = 0; u < len-namelen; - p++, u++) { - if (vct_issp(*p) || vct_isctl(*p)) + /* Check HTTP token */ + for (p = val.b; p < val.e; p++) { + if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); } - - if (!u) - return (H2SE_PROTOCOL_ERROR); - } else if (!strncmp(b, ":authority: ", namelen)) { - b+=6; - len-=6; - memcpy(b, "host", 4); - n = hp->nhd; + } else if (!Tstrcmp(nm, ":authority")) { + /* NB: we inject "host" in place of "rity" for + * the ":authority" pseudo-header. + */ + memcpy(d->out + 6, "host", 4); + hdr.b += 6; + nm = Tstr(":authority"); /* preserve original */ } else { /* Unknown pseudo-header */ VSLb(hp->vsl, SLT_BogoHeader, "Unknown pseudo-header: %.*s", - (int)(len > 20 ? 20 : len), b); + vmin_t(int, Tlen(hdr), 20), hdr.b); return (H2SE_PROTOCOL_ERROR); // rfc7540,l,2990,2992 } - } else - n = hp->nhd; + } + + if (disallow_empty && Tlen(val) == 0) { + VSLb(hp->vsl, SLT_BogoHeader, + "Empty pseudo-header %.*s", + (int)Tlen(nm), nm.b); + return (H2SE_PROTOCOL_ERROR); + } if (n < HTTP_HDR_FIRST) { - /* Check for duplicate pseudo-header */ if (hp->hd[n].b != NULL) { VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header %.*s%.*s", - (int)namelen, b0, (int)(len > 20 ? 20 : len), b); + "Duplicate pseudo-header %.*s", + (int)Tlen(nm), nm.b); return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 } } else { /* Check for space in struct http */ if (n >= hp->shd) { - VSLb(hp->vsl, SLT_LostHeader, "Too many headers: %.*s", - (int)(len > 20 ? 20 : len), b); + VSLb(hp->vsl, SLT_LostHeader, + "Too many headers: %.*s", + vmin_t(int, Tlen(hdr), 20), hdr.b); return (H2SE_ENHANCE_YOUR_CALM); } hp->nhd++; } - hp->hd[n].b = b; - hp->hd[n].e = b + len; - - if (disallow_empty && !Tlen(hp->hd[n])) { - VSLb(hp->vsl, SLT_BogoHeader, - "Empty pseudo-header %.*s", - (int)namelen, b0); - return (H2SE_PROTOCOL_ERROR); - } - + hp->hd[n] = hdr; return (0); } @@ -393,8 +403,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out_u); if (d->error) break; - d->error = h2h_addhdr(d, hp, d->out, - d->namelen, d->out_u); + d->error = h2h_addhdr(hp, d); if (d->error) break; d->out[d->out_u++] = '\0'; /* Zero guard */ From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:11 +0000 (UTC) Subject: [master] c0b452b30 http2_hpack: Refuse multiple :authority pseudo headers Message-ID: <20240404142911.D06FE10178C@lists.varnish-cache.org> commit c0b452b301a69f85ad5494eb2de99f5fc0b06e7c Author: Dridi Boukelmoune Date: Wed Mar 27 16:17:08 2024 +0100 http2_hpack: Refuse multiple :authority pseudo headers It became explicit in rfc9113: > The same pseudo-header field name MUST NOT appear more than once in a > field block. While at it, the duplicate pseudo-header error can be consolidated in a single location instead of adding one more branch. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 126a70d3a..5962ddb51 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -227,6 +227,7 @@ struct h2h_decode { unsigned magic; #define H2H_DECODE_MAGIC 0xd092bde4 + unsigned has_authority:1; unsigned has_scheme:1; h2_error error; enum vhd_ret_e vhd_ret; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 48e6e5c5c..a134c379a 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -145,7 +145,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) txt hdr, nm, val; int disallow_empty; const char *p; - unsigned n; + unsigned n, has_dup; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); h2h_assert_ready(d); @@ -162,6 +162,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) val.e = hdr.e; disallow_empty = 0; + has_dup = 0; if (Tlen(hdr) > UINT_MAX) { /* XXX: cache_param max header size */ VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", hdr.b); @@ -205,14 +206,8 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ - if (d->has_scheme) { - VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header :scheme: %.*s", - vmin_t(int, Tlen(val), 20), val.b); - return (H2SE_PROTOCOL_ERROR); - } - hdr.b++; + has_dup = d->has_scheme; d->has_scheme = 1; disallow_empty = 1; @@ -228,6 +223,8 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) memcpy(d->out + 6, "host", 4); hdr.b += 6; nm = Tstr(":authority"); /* preserve original */ + has_dup = d->has_authority; + d->has_authority = 1; } else { /* Unknown pseudo-header */ VSLb(hp->vsl, SLT_BogoHeader, @@ -244,14 +241,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) return (H2SE_PROTOCOL_ERROR); } - if (n < HTTP_HDR_FIRST) { - if (hp->hd[n].b != NULL) { - VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header %.*s", - (int)Tlen(nm), nm.b); - return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 - } - } else { + if (n >= HTTP_HDR_FIRST) { /* Check for space in struct http */ if (n >= hp->shd) { VSLb(hp->vsl, SLT_LostHeader, @@ -260,6 +250,15 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) return (H2SE_ENHANCE_YOUR_CALM); } hp->nhd++; + AZ(hp->hd[n].b); + } + + if (has_dup || hp->hd[n].b != NULL) { + assert(nm.b[0] == ':'); + VSLb(hp->vsl, SLT_BogoHeader, + "Duplicate pseudo-header %.*s", + (int)Tlen(nm), nm.b); + return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 } hp->hd[n] = hdr; diff --git a/bin/varnishtest/tests/r02351.vtc b/bin/varnishtest/tests/r02351.vtc index 09d12e541..936f522b9 100644 --- a/bin/varnishtest/tests/r02351.vtc +++ b/bin/varnishtest/tests/r02351.vtc @@ -1,4 +1,4 @@ -varnishtest "#2351: :path/:method error handling" +varnishtest "#2351: h2 pseudo-headers error handling" server s1 { rxreq @@ -43,6 +43,16 @@ client c1 { } -run } -run +client c2 { + # Duplicate :authority + stream next { + txreq -noadd -hdr :path / -hdr :method GET -hdr :scheme http \ + -hdr :authority example.com -hdr :authority example.org + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + varnish v1 -expect MEMPOOL.req0.live == 0 varnish v1 -expect MEMPOOL.req1.live == 0 varnish v1 -expect MEMPOOL.sess0.live == 0 From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:11 +0000 (UTC) Subject: [master] d9928c3bc http2_hpack: Remove one level of nesting Message-ID: <20240404142911.EA2F3101790@lists.varnish-cache.org> commit d9928c3bc3dd5a9be3e36363ce16e64db536f00e Author: Dridi Boukelmoune Date: Wed Mar 27 16:28:18 2024 +0100 http2_hpack: Remove one level of nesting Better diff with the --ignore-all-space option. diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index a134c379a..c2013c1f2 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -169,69 +169,64 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) return (H2SE_ENHANCE_YOUR_CALM); } - if (*nm.b == ':') { - /* Match H/2 pseudo headers */ - /* XXX: Should probably have some include tbl for - pseudo-headers */ - if (!Tstrcmp(nm, ":method")) { - hdr.b = val.b; - n = HTTP_HDR_METHOD; - disallow_empty = 1; - - /* Check HTTP token */ - for (p = hdr.b; p < hdr.e; p++) { - if (!vct_istchar(*p)) - return (H2SE_PROTOCOL_ERROR); - } - } else if (!Tstrcmp(nm, ":path")) { - hdr.b = val.b; - n = HTTP_HDR_URL; - disallow_empty = 1; - - // rfc9113,l,2693,2705 - if (Tlen(val) > 0 && *val.b != '/' && - Tstrcmp(val, "*")) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal :path pseudo-header %.*s", - (int)Tlen(val), val.b); + /* Match H/2 pseudo headers */ + /* XXX: Should probably have some include tbl for pseudo-headers */ + if (!Tstrcmp(nm, ":method")) { + hdr.b = val.b; + n = HTTP_HDR_METHOD; + disallow_empty = 1; + + /* Check HTTP token */ + for (p = hdr.b; p < hdr.e; p++) { + if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); - } + } + } else if (!Tstrcmp(nm, ":path")) { + hdr.b = val.b; + n = HTTP_HDR_URL; + disallow_empty = 1; - /* Path cannot contain LWS or CTL */ - for (p = hdr.b; p < hdr.e; p++) { - if (vct_islws(*p) || vct_isctl(*p)) - return (H2SE_PROTOCOL_ERROR); - } - } else if (!Tstrcmp(nm, ":scheme")) { - /* XXX: What to do about this one? (typically - "http" or "https"). For now set it as a normal - header, stripping the first ':'. */ - hdr.b++; - has_dup = d->has_scheme; - d->has_scheme = 1; - disallow_empty = 1; - - /* Check HTTP token */ - for (p = val.b; p < val.e; p++) { - if (!vct_istchar(*p)) - return (H2SE_PROTOCOL_ERROR); - } - } else if (!Tstrcmp(nm, ":authority")) { - /* NB: we inject "host" in place of "rity" for - * the ":authority" pseudo-header. - */ - memcpy(d->out + 6, "host", 4); - hdr.b += 6; - nm = Tstr(":authority"); /* preserve original */ - has_dup = d->has_authority; - d->has_authority = 1; - } else { - /* Unknown pseudo-header */ + // rfc9113,l,2693,2705 + if (Tlen(val) > 0 && val.b[0] != '/' && Tstrcmp(val, "*")) { VSLb(hp->vsl, SLT_BogoHeader, - "Unknown pseudo-header: %.*s", - vmin_t(int, Tlen(hdr), 20), hdr.b); - return (H2SE_PROTOCOL_ERROR); // rfc7540,l,2990,2992 + "Illegal :path pseudo-header %.*s", + (int)Tlen(val), val.b); + return (H2SE_PROTOCOL_ERROR); } + + /* Path cannot contain LWS or CTL */ + for (p = hdr.b; p < hdr.e; p++) { + if (vct_islws(*p) || vct_isctl(*p)) + return (H2SE_PROTOCOL_ERROR); + } + } else if (!Tstrcmp(nm, ":scheme")) { + /* XXX: What to do about this one? (typically + "http" or "https"). For now set it as a normal + header, stripping the first ':'. */ + hdr.b++; + has_dup = d->has_scheme; + d->has_scheme = 1; + disallow_empty = 1; + + /* Check HTTP token */ + for (p = val.b; p < val.e; p++) { + if (!vct_istchar(*p)) + return (H2SE_PROTOCOL_ERROR); + } + } else if (!Tstrcmp(nm, ":authority")) { + /* NB: we inject "host" in place of "rity" for + * the ":authority" pseudo-header. + */ + memcpy(d->out + 6, "host", 4); + hdr.b += 6; + nm = Tstr(":authority"); /* preserve original */ + has_dup = d->has_authority; + d->has_authority = 1; + } else if (nm.b[0] == ':') { + VSLb(hp->vsl, SLT_BogoHeader, + "Unknown pseudo-header: %.*s", + vmin_t(int, Tlen(hdr), 20), hdr.b); + return (H2SE_PROTOCOL_ERROR); // rfc7540,l,2990,2992 } if (disallow_empty && Tlen(val) == 0) { From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:12 +0000 (UTC) Subject: [master] cdcae28a9 http2_hpack: Also rewrite h2h_checkhdr() for clarity Message-ID: <20240404142912.26DAD101798@lists.varnish-cache.org> commit cdcae28a960be46ddf9cc77c9d70bc9ceae873c5 Author: Dridi Boukelmoune Date: Thu Mar 28 14:13:36 2024 +0100 http2_hpack: Also rewrite h2h_checkhdr() for clarity It does a first pass on header names and values, and only logs errors, so the signature is updated accordingly and the call site is moved into h2h_addhdr(). diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index c2013c1f2..531c50307 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -53,9 +53,10 @@ h2h_assert_ready(struct h2h_decode *d) // rfc9113,l,2493,2528 static h2_error -h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) +h2h_checkhdr(struct vsl_log *vsl, txt nm, txt val) { const char *p; + int l; enum { FLD_NAME_FIRST, FLD_NAME, @@ -63,23 +64,17 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) FLD_VALUE } state; - CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - AN(b); - assert(namelen >= 2); /* 2 chars from the ': ' that we added */ - assert(namelen <= len); - assert(b[namelen - 2] == ':'); - assert(b[namelen - 1] == ' '); - - if (namelen == 2) { - VSLb(hp->vsl, SLT_BogoHeader, "Empty name"); + if (Tlen(nm) == 0) { + VSLb(vsl, SLT_BogoHeader, "Empty name"); return (H2SE_PROTOCOL_ERROR); } - // VSLb(hp->vsl, SLT_Debug, "CHDR [%.*s] [%.*s]", - // (int)namelen, b, (int)(len - namelen), b + namelen); + // VSLb(vsl, SLT_Debug, "CHDR [%.*s] [%.*s]", + // (int)Tlen(nm), nm.b, (int)Tlen(val), val.b); + l = vmin_t(int, Tlen(nm) + 2 + Tlen(val), 20); state = FLD_NAME_FIRST; - for (p = b; p < b + namelen - 2; p++) { + for (p = nm.b; p < nm.e; p++) { switch(state) { case FLD_NAME_FIRST: state = FLD_NAME; @@ -88,15 +83,15 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) /* FALL_THROUGH */ case FLD_NAME: if (isupper(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, + VSLb(vsl, SLT_BogoHeader, "Illegal field header name (upper-case): %.*s", - (int)(len > 20 ? 20 : len), b); + l, nm.b); return (H2SE_PROTOCOL_ERROR); } if (!vct_istchar(*p) || *p == ':') { - VSLb(hp->vsl, SLT_BogoHeader, + VSLb(vsl, SLT_BogoHeader, "Illegal field header name (non-token): %.*s", - (int)(len > 20 ? 20 : len), b); + l, nm.b); return (H2SE_PROTOCOL_ERROR); } break; @@ -106,22 +101,20 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } state = FLD_VALUE_FIRST; - for (p = b + namelen; p < b + len; p++) { + for (p = val.b; p < val.e; p++) { switch(state) { case FLD_VALUE_FIRST: if (vct_issp(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field value start %.*s", - (int)(len > 20 ? 20 : len), b); + VSLb(vsl, SLT_BogoHeader, + "Illegal field value start %.*s", l, nm.b); return (H2SE_PROTOCOL_ERROR); } state = FLD_VALUE; /* FALL_THROUGH */ case FLD_VALUE: if (!vct_ishdrval(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field value %.*s", - (int)(len > 20 ? 20 : len), b); + VSLb(vsl, SLT_BogoHeader, + "Illegal field value %.*s", l, nm.b); return (H2SE_PROTOCOL_ERROR); } break; @@ -129,10 +122,9 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) WRONG("http2 field value validation state"); } } - if (state == FLD_VALUE && vct_issp(b[len - 1])) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field value (end) %.*s", - (int)(len > 20 ? 20 : len), b); + if (state == FLD_VALUE && vct_issp(val.e[-1])) { + VSLb(vsl, SLT_BogoHeader, + "Illegal field value (end) %.*s", l, nm.b); return (H2SE_PROTOCOL_ERROR); } return (0); @@ -146,6 +138,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) int disallow_empty; const char *p; unsigned n, has_dup; + h2_error err; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); h2h_assert_ready(d); @@ -161,6 +154,10 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) val.b = nm.e + 2; val.e = hdr.e; + err = h2h_checkhdr(hp->vsl, nm, val); + if (err != NULL) + return (err); + disallow_empty = 0; has_dup = 0; @@ -393,10 +390,6 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->error = H2SE_ENHANCE_YOUR_CALM; break; } - d->error = h2h_checkhdr(hp, d->out, d->namelen, - d->out_u); - if (d->error) - break; d->error = h2h_addhdr(hp, d); if (d->error) break; From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:12 +0000 (UTC) Subject: [master] a90f5425e http2_hpack: Enforce http_req_hdr_len limit Message-ID: <20240404142912.463A51017A6@lists.varnish-cache.org> commit a90f5425e7b47420441ec4bd220c397de4739de0 Author: Dridi Boukelmoune Date: Thu Mar 28 15:21:01 2024 +0100 http2_hpack: Enforce http_req_hdr_len limit Refs #3709 diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 531c50307..b47dba507 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -161,7 +161,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) disallow_empty = 0; has_dup = 0; - if (Tlen(hdr) > UINT_MAX) { /* XXX: cache_param max header size */ + if (Tlen(hdr) > cache_param->http_req_hdr_len) { VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", hdr.b); return (H2SE_ENHANCE_YOUR_CALM); } diff --git a/bin/varnishtest/tests/r03709.vtc b/bin/varnishtest/tests/r03709.vtc new file mode 100644 index 000000000..7439efba3 --- /dev/null +++ b/bin/varnishtest/tests/r03709.vtc @@ -0,0 +1,21 @@ +varnishtest "h2 req limits" + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set http_req_hdr_len 40b" +varnish v1 -vcl { + backend be none; +} -start + +logexpect l1 -v v1 -g raw -q BogoHeader { + expect 0 1001 BogoHeader "Header too large: :path" +} -start + +client c1 { + stream next { + txreq -url ${string,repeat,4,/123456789} + rxrst + expect rst.err == ENHANCE_YOUR_CALM + } -run +} -run + +logexpect l1 -wait From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:12 +0000 (UTC) Subject: [master] 803140a9a http2: New H2_ERROR_MATCH() helper macro Message-ID: <20240404142912.664041017B2@lists.varnish-cache.org> commit 803140a9a7609ea745c2a929c8474b9a4076a6fd Author: Dridi Boukelmoune Date: Thu Mar 28 15:56:21 2024 +0100 http2: New H2_ERROR_MATCH() helper macro diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 5962ddb51..9fbb58443 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -50,6 +50,9 @@ struct h2_error_s { typedef const struct h2_error_s *h2_error; +#define H2_ERROR_MATCH(err, target) \ + ((err) != NULL && (err)->val == (target)->val) + #define H2_CUSTOM_ERRORS #define H2EC1(U,v,g,r,d) extern const struct h2_error_s H2CE_##U[1]; #define H2EC2(U,v,g,r,d) extern const struct h2_error_s H2SE_##U[1]; diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 316d13dc5..4b072f532 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -435,6 +435,6 @@ H2_Send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, h2_send(wrk, r2, ftyp, flags, len, ptr, counter); h2e = h2_errcheck(r2, r2->h2sess); - if (h2e != NULL && h2e->val == H2SE_CANCEL->val) + if (H2_ERROR_MATCH(h2e, H2SE_CANCEL)) H2_Send_RST(wrk, r2->h2sess, r2, r2->stream, h2e); } From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:12 +0000 (UTC) Subject: [master] 97e83ddb3 http2_hpack: New custom h2 error for http_req_size Message-ID: <20240404142912.7F4B71017BF@lists.varnish-cache.org> commit 97e83ddb30903a4ff8843d2f44bcec075f96ff71 Author: Dridi Boukelmoune Date: Thu Mar 28 16:02:53 2024 +0100 http2_hpack: New custom h2 error for http_req_size diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index b47dba507..92858543e 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -343,7 +343,8 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) /* Only H2E_ENHANCE_YOUR_CALM indicates that we should continue processing. Other errors should have been returned and handled by the caller. */ - assert(d->error == 0 || d->error == H2SE_ENHANCE_YOUR_CALM); + if (d->error != NULL) + assert(H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)); while (1) { AN(d->out); @@ -362,7 +363,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; } - if (d->error == H2SE_ENHANCE_YOUR_CALM) { + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { d->out_u = 0; assert(d->out_u < d->out_l); continue; @@ -374,7 +375,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) case VHD_NAME: assert(d->namelen == 0); if (d->out_l - d->out_u < 2) { - d->error = H2SE_ENHANCE_YOUR_CALM; + d->error = H2SE_REQ_SIZE; break; } d->out[d->out_u++] = ':'; @@ -387,7 +388,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) case VHD_VALUE: assert(d->namelen > 0); if (d->out_l - d->out_u < 1) { - d->error = H2SE_ENHANCE_YOUR_CALM; + d->error = H2SE_REQ_SIZE; break; } d->error = h2h_addhdr(hp, d); @@ -401,7 +402,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; case VHD_BUF: - d->error = H2SE_ENHANCE_YOUR_CALM; + d->error = H2SE_REQ_SIZE; break; default: @@ -409,7 +410,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; } - if (d->error == H2SE_ENHANCE_YOUR_CALM) { + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { d->out = d->reset; d->out_l = e - d->out; d->out_u = 0; @@ -418,7 +419,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; } - if (d->error == H2SE_ENHANCE_YOUR_CALM) + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) return (0); /* Stream error, delay reporting until h2h_decode_fini so that we can process the complete header block */ diff --git a/include/tbl/h2_error.h b/include/tbl/h2_error.h index f4dc5ca2c..adfbbde42 100644 --- a/include/tbl/h2_error.h +++ b/include/tbl/h2_error.h @@ -197,6 +197,15 @@ H2_ERROR( /* reason */ SC_BANKRUPT, /* descr */ "http/2 bankrupt connection" ) + +H2_ERROR( + /* name */ REQ_SIZE, + /* val */ 11, /* ENHANCE_YOUR_CALM */ + /* types */ 2, + /* goaway */ 0, + /* reason */ SC_NULL, + /* descr */ "HTTP/2 header list exceeded http_req_size" +) # undef H2_CUSTOM_ERRORS #endif From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:12 +0000 (UTC) Subject: [master] 7ccffe7bb http2_hpack: Enforce http_req_size limit Message-ID: <20240404142912.A0FE41017CF@lists.varnish-cache.org> commit 7ccffe7bb03a4fe3b90181c217b54ca7fe7201a0 Author: Dridi Boukelmoune Date: Thu Mar 28 16:08:46 2024 +0100 http2_hpack: Enforce http_req_size limit Fixes #3709 Closes #3892 diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 92858543e..4d155e3d1 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -269,7 +269,8 @@ h2h_decode_init(const struct h2_sess *h2) d = h2->decode; INIT_OBJ(d, H2H_DECODE_MAGIC); VHD_Init(d->vhd); - d->out_l = WS_ReserveAll(h2->new_req->http->ws); + d->out_l = WS_ReserveSize(h2->new_req->http->ws, + cache_param->http_req_size); /* * Can't do any work without any buffer * space. Require non-zero size. @@ -310,6 +311,10 @@ h2h_decode_fini(const struct h2_sess *h2) } else ret = d->error; FINI_OBJ(d); + if (ret == H2SE_REQ_SIZE) { + VSLb(h2->new_req->http->vsl, SLT_LostHeader, + "Header list too large"); + } return (ret); } diff --git a/bin/varnishtest/tests/r03709.vtc b/bin/varnishtest/tests/r03709.vtc index 7439efba3..242afe2f1 100644 --- a/bin/varnishtest/tests/r03709.vtc +++ b/bin/varnishtest/tests/r03709.vtc @@ -2,17 +2,40 @@ varnishtest "h2 req limits" varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set http_req_hdr_len 40b" +varnish v1 -cliok "param.set http_req_size 512b" varnish v1 -vcl { backend be none; } -start -logexpect l1 -v v1 -g raw -q BogoHeader { +logexpect l1 -v v1 -g raw -q BogoHeader,LostHeader { expect 0 1001 BogoHeader "Header too large: :path" + expect 0 1002 LostHeader "Header list too large" } -start client c1 { stream next { - txreq -url ${string,repeat,4,/123456789} + txreq -url ${string,repeat,4,/123456789} \ + -hdr limit http_req_hdr_len + rxrst + expect rst.err == ENHANCE_YOUR_CALM + } -run + + stream next { + txreq -url "/http_req_size" \ + -hdr hdr1 ${string,repeat,3,/123456789} \ + -hdr hdr2 ${string,repeat,3,/123456789} \ + -hdr hdr3 ${string,repeat,3,/123456789} \ + -hdr hdr4 ${string,repeat,3,/123456789} \ + -hdr hdr5 ${string,repeat,3,/123456789} \ + -hdr hdr6 ${string,repeat,3,/123456789} \ + -hdr hdr7 ${string,repeat,3,/123456789} \ + -hdr hdr8 ${string,repeat,3,/123456789} \ + -hdr hdr9 ${string,repeat,3,/123456789} \ + -hdr hdr10 ${string,repeat,3,/123456789} \ + -hdr hdr11 ${string,repeat,3,/123456789} \ + -hdr hdr12 ${string,repeat,3,/123456789} \ + -hdr hdr13 ${string,repeat,3,/123456789} \ + -hdr hdr14 ${string,repeat,3,/123456789} rxrst expect rst.err == ENHANCE_YOUR_CALM } -run From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:12 +0000 (UTC) Subject: [master] af1a69838 vtc: Add http_max_hdr coverage to r3709 Message-ID: <20240404142912.C07D71017EA@lists.varnish-cache.org> commit af1a698380da6c46c2e15d80327f00af91c011a5 Author: Dridi Boukelmoune Date: Thu Mar 28 16:22:59 2024 +0100 vtc: Add http_max_hdr coverage to r3709 For the sole purpose of having these limits tested in a single place. diff --git a/bin/varnishtest/tests/r03709.vtc b/bin/varnishtest/tests/r03709.vtc index 242afe2f1..65038fcce 100644 --- a/bin/varnishtest/tests/r03709.vtc +++ b/bin/varnishtest/tests/r03709.vtc @@ -3,6 +3,7 @@ varnishtest "h2 req limits" varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set http_req_hdr_len 40b" varnish v1 -cliok "param.set http_req_size 512b" +varnish v1 -cliok "param.set http_max_hdr 32" varnish v1 -vcl { backend be none; } -start @@ -10,6 +11,7 @@ varnish v1 -vcl { logexpect l1 -v v1 -g raw -q BogoHeader,LostHeader { expect 0 1001 BogoHeader "Header too large: :path" expect 0 1002 LostHeader "Header list too large" + expect 0 1003 LostHeader "Too many headers" } -start client c1 { @@ -39,6 +41,41 @@ client c1 { rxrst expect rst.err == ENHANCE_YOUR_CALM } -run + + stream next { + txreq -url "/http_max_hdr" \ + -hdr hdr1 val1 \ + -hdr hdr2 val2 \ + -hdr hdr3 val3 \ + -hdr hdr4 val4 \ + -hdr hdr4 val4 \ + -hdr hdr5 val5 \ + -hdr hdr6 val6 \ + -hdr hdr7 val7 \ + -hdr hdr8 val8 \ + -hdr hdr9 val9 \ + -hdr hdr10 val10 \ + -hdr hdr11 val11 \ + -hdr hdr11 val11 \ + -hdr hdr11 val11 \ + -hdr hdr12 val12 \ + -hdr hdr13 val13 \ + -hdr hdr14 val14 \ + -hdr hdr15 val15 \ + -hdr hdr16 val16 \ + -hdr hdr17 val17 \ + -hdr hdr18 val18 \ + -hdr hdr19 val19 \ + -hdr hdr20 val20 \ + -hdr hdr20 val20 \ + -hdr hdr21 val21 \ + -hdr hdr22 val22 \ + -hdr hdr23 val23 \ + -hdr hdr24 val24 \ + -hdr hdr25 val25 + rxrst + expect rst.err == ENHANCE_YOUR_CALM + } -run } -run logexpect l1 -wait From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:12 +0000 (UTC) Subject: [master] f7b31d651 param: Document mapping to HTTP/2 settings Message-ID: <20240404142912.E04661017FA@lists.varnish-cache.org> commit f7b31d651d36cce5066f055c2e3fe460e1789993 Author: Dridi Boukelmoune Date: Thu Mar 28 16:34:35 2024 +0100 param: Document mapping to HTTP/2 settings With the exception of h2_max_header_list_size that is not advertised as such despite being ent as part of the initial SETTINGS frame. The same parameter also sees its default and maximum values updated to 2^32-1. This is based on this sentence from rfc9113: > The initial value of this setting is unlimited. This aligns the h2_max_header_list_size parameter with the values set in h2_settings.h for MAX_HEADER_LIST_SIZE. diff --git a/include/tbl/params.h b/include/tbl/params.h index 42d72947b..647ff7e87 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1219,6 +1219,12 @@ PARAM_SIMPLE( /* flags */ WIZARD ) +#define H2_SETTING_NAME(nm) "SETTINGS_" #nm +#define H2_SETTING_DESCR(nm) \ + "\n\nThe value of this parameter defines " H2_SETTING_NAME(nm) \ + " in the initial SETTINGS frame sent to the client when a new " \ + "HTTP2 session is established." + PARAM_SIMPLE( /* name */ h2_header_table_size, /* type */ bytes_u, @@ -1230,6 +1236,7 @@ PARAM_SIMPLE( "HTTP2 header table size.\n" "This is the size that will be used for the HPACK dynamic\n" "decoding table." + H2_SETTING_DESCR(HEADER_TABLE_SIZE) ) PARAM_SIMPLE( @@ -1243,6 +1250,7 @@ PARAM_SIMPLE( "HTTP2 Maximum number of concurrent streams.\n" "This is the number of requests that can be active\n" "at the same time for a single HTTP2 connection." + H2_SETTING_DESCR(MAX_CONCURRENT_STREAMS) ) /* We have a strict min at the protocol default here. This is because we @@ -1258,7 +1266,8 @@ PARAM_SIMPLE( /* def */ "65535b", /* units */ "bytes", /* descr */ - "HTTP2 initial flow control window size.", + "HTTP2 initial flow control window size." + H2_SETTING_DESCR(INITIAL_WINDOW_SIZE) ) PARAM_SIMPLE( @@ -1270,19 +1279,23 @@ PARAM_SIMPLE( /* units */ "bytes", /* descr */ "HTTP2 maximum per frame payload size we are willing to accept." + H2_SETTING_DESCR(MAX_FRAME_SIZE) ) PARAM_SIMPLE( /* name */ h2_max_header_list_size, /* type */ bytes_u, /* min */ "0b", - /* max */ NULL, - /* def */ "2147483647b", + /* max */ "4294967295b", + /* def */ "4294967295b", /* units */ "bytes", /* descr */ "HTTP2 maximum size of an uncompressed header list." ) +#undef H2_SETTING_DESCR +#undef H2_SETTING_NAME + #define H2_RR_INFO \ "Changes to this parameter affect the default for new HTTP2 " \ "sessions. vmod_h2(3) can be used to adjust it from VCL." From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:13 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:13 +0000 (UTC) Subject: [master] f1fb85fed http2_session: Advertise http_req_size to clients Message-ID: <20240404142913.37590101806@lists.varnish-cache.org> commit f1fb85fed1f943a2005d7da06de94ff6fa6cc0e2 Author: Dridi Boukelmoune Date: Thu Mar 28 16:35:52 2024 +0100 http2_session: Advertise http_req_size to clients Since http_req_size was already established for this purpose, and is now enforced for h2 traffic, it should naturally become the basis for the MAX_HEADER_LIST_SIZE setting in the initial SETTINGS frame sent to clients. The h2_max_header_list_size parameter will grow a new purpose. diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index acbfdbb51..605798810 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -85,6 +85,7 @@ h2_local_settings(struct h2_settings *h2s) h2s->l = cache_param->h2_##l; #include "tbl/h2_settings.h" #undef H2_SETTINGS_PARAM_ONLY + h2s->max_header_list_size = cache_param->http_req_size; } /********************************************************************** diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index a139cc33a..97120a9e7 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -69,7 +69,7 @@ varnish v1 -expect MEMPOOL.sess1.live == 0 process p1 -stop # shell {cat ${tmpdir}/vlog} # SETTINGS with default initial window size -shell -match {1001 H2TxHdr c \[000006040000000000\]} { +shell -match {1001 H2TxHdr c \[00000c040000000000\]} { cat ${tmpdir}/vlog } diff --git a/bin/varnishtest/tests/t02005.vtc b/bin/varnishtest/tests/t02005.vtc index c5176f725..39737f93a 100644 --- a/bin/varnishtest/tests/t02005.vtc +++ b/bin/varnishtest/tests/t02005.vtc @@ -32,7 +32,7 @@ varnish v1 -cliok "param.set debug +syncvsl" logexpect l1 -v v1 -g raw { expect * 1001 ReqAcct "80 7 87 78 8 86" - expect * 1000 ReqAcct "45 8 53 63 28 91" + expect * 1000 ReqAcct "45 8 53 63 34 97" } -start client c1 { diff --git a/include/tbl/h2_settings.h b/include/tbl/h2_settings.h index 6c4520711..2dbac671f 100644 --- a/include/tbl/h2_settings.h +++ b/include/tbl/h2_settings.h @@ -92,6 +92,7 @@ H2_SETTING( // rfc7540,l,2150,2157 H2CE_PROTOCOL_ERROR ) +#ifndef H2_SETTINGS_PARAM_ONLY H2_SETTING( // rfc7540,l,2159,2167 MAX_HEADER_LIST_SIZE, max_header_list_size, @@ -101,6 +102,7 @@ H2_SETTING( // rfc7540,l,2159,2167 0xffffffff, 0 ) +#endif #undef H2_SETTING /*lint -restore */ diff --git a/include/tbl/params.h b/include/tbl/params.h index 647ff7e87..124826805 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -605,11 +605,13 @@ PARAM_SIMPLE( /* units */ "bytes", /* descr */ "Maximum number of bytes of HTTP client request we will deal with. " - " This is a limit on all bytes up to the double blank line which " - "ends the HTTP request.\n" + "This is a limit on all bytes up to the double blank line which " + "ends the HTTP request. " "The memory for the request is allocated from the client workspace " "(param: workspace_client) and this parameter limits how much of " - "that the request is allowed to take up." + "that the request is allowed to take up.\n\n" + "For HTTP2 clients, it is advertised as MAX_HEADER_LIST_SIZE in " + "the initial SETTINGS frame." ) PARAM_SIMPLE( @@ -1290,7 +1292,9 @@ PARAM_SIMPLE( /* def */ "4294967295b", /* units */ "bytes", /* descr */ - "HTTP2 maximum size of an uncompressed header list." + "HTTP2 maximum size of an uncompressed header list. This parameter " + "is not mapped to " H2_SETTING_NAME(MAX_HEADER_LIST_SIZE) " in the " + "initial SETTINGS frame, the http_req_size parameter is instead." ) #undef H2_SETTING_DESCR From dridi.boukelmoune at gmail.com Thu Apr 4 14:29:13 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:29:13 +0000 (UTC) Subject: [master] 0e75d4635 http2_hpack: Enforce h2_max_header_list_size Message-ID: <20240404142913.5656810180F@lists.varnish-cache.org> commit 0e75d46357fc26ab59b9f660460d7c748f2c8be4 Author: Dridi Boukelmoune Date: Thu Mar 28 16:50:39 2024 +0100 http2_hpack: Enforce h2_max_header_list_size This parameter has a new role that consists in interrupting connections when decoding an HPACK block leads to a header list so large that the client must be stopped. By default, too large is 150% of http_req_size. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 9fbb58443..89e32309c 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -236,6 +236,7 @@ struct h2h_decode { enum vhd_ret_e vhd_ret; char *out; char *reset; + int64_t limit; size_t out_l; size_t out_u; size_t namelen; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 4d155e3d1..c4274a656 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -278,6 +278,14 @@ h2h_decode_init(const struct h2_sess *h2) XXXAN(d->out_l); d->out = WS_Reservation(h2->new_req->http->ws); d->reset = d->out; + + if (cache_param->h2_max_header_list_size == 0) + d->limit = h2->local_settings.max_header_list_size * 1.5; + else + d->limit = cache_param->h2_max_header_list_size; + + if (d->limit < h2->local_settings.max_header_list_size) + d->limit = INT64_MAX; } /* Possible error returns: @@ -351,7 +359,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) if (d->error != NULL) assert(H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)); - while (1) { + while (d->limit >= 0) { AN(d->out); assert(d->out_u <= d->out_l); d->vhd_ret = VHD_Decode(d->vhd, h2->dectbl, in, in_l, &in_u, @@ -369,6 +377,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) } if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { + d->limit -= d->out_u; d->out_u = 0; assert(d->out_u < d->out_l); continue; @@ -402,6 +411,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out[d->out_u++] = '\0'; /* Zero guard */ d->out += d->out_u; d->out_l -= d->out_u; + d->limit -= d->out_u; d->out_u = 0; d->namelen = 0; break; @@ -418,15 +428,25 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { d->out = d->reset; d->out_l = e - d->out; + d->limit -= d->out_u; d->out_u = 0; assert(d->out_l > 0); } else if (d->error) break; } - if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) - return (0); /* Stream error, delay reporting until - h2h_decode_fini so that we can process the - complete header block */ + if (d->limit < 0) { + /* Fatal error, the client exceeded both http_req_size + * and h2_max_header_list_size. */ + VSLb(h2->vsl, SLT_SessError, "Header list too large"); + return (H2CE_ENHANCE_YOUR_CALM); + } + + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { + /* Stream error, delay reporting until h2h_decode_fini so + * that we can process the complete header block. */ + return (NULL); + } + return (d->error); } diff --git a/bin/varnishtest/tests/f00015.vtc b/bin/varnishtest/tests/f00015.vtc new file mode 100644 index 000000000..de5df8ed2 --- /dev/null +++ b/bin/varnishtest/tests/f00015.vtc @@ -0,0 +1,19 @@ +varnishtest "h2 CONTINUATION flood" + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set vsl_mask -H2RxHdr,-H2RxBody" +varnish v1 -vcl { backend be none; } -start + +client c1 { + non_fatal + stream next { + txreq -nohdrend + loop 1000 { + txcont -nohdrend -hdr noise ${string,repeat,4096,x} + } + txcont -hdr last header + } -run +} -run + +varnish v1 -expect MAIN.s_req_hdrbytes < 65536 +varnish v1 -expect MAIN.sc_overload == 1 diff --git a/include/tbl/params.h b/include/tbl/params.h index 124826805..1b50d4cda 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1289,12 +1289,20 @@ PARAM_SIMPLE( /* type */ bytes_u, /* min */ "0b", /* max */ "4294967295b", - /* def */ "4294967295b", + /* def */ "0b", /* units */ "bytes", /* descr */ "HTTP2 maximum size of an uncompressed header list. This parameter " "is not mapped to " H2_SETTING_NAME(MAX_HEADER_LIST_SIZE) " in the " - "initial SETTINGS frame, the http_req_size parameter is instead." + "initial SETTINGS frame, the http_req_size parameter is instead.\n\n" + "The http_req_size advises HTTP2 clients of the maximum size for " + "the header list. Exceeding http_req_size results in a reset stream " + "after processing the HPACK block to perserve the connection, but " + "exceeding h2_max_header_list_size results in the HTTP2 connection " + "going away immediately.\n\n" + "If h2_max_header_list_size is lower than http_req_size, it has no " + "effect, except for the special value zero interpreted as 150% of " + "http_req_size." ) #undef H2_SETTING_DESCR From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:09 +0000 (UTC) Subject: [7.4] 33df4b371 h2: the :scheme pseudo header is not optional Message-ID: <20240404143009.7FE831020AB@lists.varnish-cache.org> commit 33df4b371eb048aafa2cf83c992877f6aee848fd Author: Asad Sajjad Ahmed Date: Wed Sep 28 14:58:38 2022 +0200 h2: the :scheme pseudo header is not optional The :scheme pseudo header is not optional in H/2 except when doing CONNECT. There is also a strict requirement for it appear only once. Signed-off-by: Asad Sajjad Ahmed Conflicts: bin/varnishtest/tests/t02025.vtc diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index ac9e2c5a9..d9c4abbb8 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -222,11 +222,14 @@ vtr_deliver_f h2_deliver; vtr_minimal_response_f h2_minimal_response; #endif /* TRANSPORT_MAGIC */ +#define H2H_DECODE_FLAG_SCHEME_SEEN 0x1 + /* http2/cache_http2_hpack.c */ struct h2h_decode { unsigned magic; #define H2H_DECODE_MAGIC 0xd092bde4 + int flags; h2_error error; enum vhd_ret_e vhd_ret; char *out; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index b5451eb22..472b86294 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -127,7 +127,7 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } static h2_error -h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) +h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) { /* XXX: This might belong in cache/cache_http.c */ const char *b0; @@ -188,9 +188,18 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ + if (*flags & H2H_DECODE_FLAG_SCHEME_SEEN) { + VSLb(hp->vsl, SLT_BogoHeader, + "Duplicate pseudo-header %.*s%.*s", + (int)namelen, b0, + (int)(len > 20 ? 20 : len), b); + return (H2SE_PROTOCOL_ERROR); + } + b++; len-=1; n = hp->nhd; + *flags |= H2H_DECODE_FLAG_SCHEME_SEEN; for (p = b + namelen, u = 0; u < len-namelen; p++, u++) { @@ -380,7 +389,8 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out_u); if (d->error) break; - d->error = h2h_addhdr(hp, d->out, d->namelen, d->out_u); + d->error = h2h_addhdr(hp, d->out, d->namelen, d->out_u, + &d->flags); if (d->error) break; d->out[d->out_u++] = '\0'; /* Zero guard */ diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 5218ed8e5..593abffbb 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -627,11 +627,13 @@ static h2_error h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { + int scheme_seen; h2_error h2e; ssize_t cl; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); + scheme_seen = h2->decode->flags & H2H_DECODE_FLAG_SCHEME_SEEN; h2e = h2h_decode_fini(h2); h2->new_req = NULL; if (h2e != NULL) { @@ -682,10 +684,17 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, VSLb(h2->vsl, SLT_Debug, "Missing :method"); return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 } + if (req->http->hd[HTTP_HDR_URL].b == NULL) { VSLb(h2->vsl, SLT_Debug, "Missing :path"); return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 } + + if (!(scheme_seen)) { + VSLb(h2->vsl, SLT_Debug, "Missing :scheme"); + return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 + } + AN(req->http->hd[HTTP_HDR_PROTO].b); if (*req->http->hd[HTTP_HDR_URL].b == '*' && diff --git a/bin/varnishtest/tests/t02026.vtc b/bin/varnishtest/tests/t02026.vtc new file mode 100644 index 000000000..b464f8cdc --- /dev/null +++ b/bin/varnishtest/tests/t02026.vtc @@ -0,0 +1,48 @@ +varnishtest "Dublicate pseudo-headers" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -arg "-p feature=+http2" -vcl+backend { +} -start + +#client c1 { +# txreq -url "/some/path" -url "/some/other/path" +# rxresp +# expect resp.status == 400 +#} -run + +#client c1 { +# txreq -req "GET" -req "POST" +# rxresp +# expect resp.status == 400 +#} -run + +#client c1 { +# txreq -proto "HTTP/1.1" -proto "HTTP/2.0" +# rxresp +# expect resp.status == 400 +#} -run + +client c1 { + stream 1 { + txreq -url "/some/path" -url "/some/other/path" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme "http" -scheme "https" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req "GET" -req "POST" + rxrst + } -run +} -run From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:09 +0000 (UTC) Subject: [7.4] 0f2060c57 h2: Manage missing :scheme as a custom error Message-ID: <20240404143009.AFFC31020B7@lists.varnish-cache.org> commit 0f2060c577d1de31d031d6e636531324df740a5a Author: Dridi Boukelmoune Date: Mon Dec 4 18:30:06 2023 +0100 h2: Manage missing :scheme as a custom error There is room for further improvement in the dynamic between HPACK and the HTTP/2 session, but this will serve as the first step. Conflicts: include/tbl/h2_error.h diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index d9c4abbb8..126a70d3a 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -222,14 +222,12 @@ vtr_deliver_f h2_deliver; vtr_minimal_response_f h2_minimal_response; #endif /* TRANSPORT_MAGIC */ -#define H2H_DECODE_FLAG_SCHEME_SEEN 0x1 - /* http2/cache_http2_hpack.c */ struct h2h_decode { unsigned magic; #define H2H_DECODE_MAGIC 0xd092bde4 - int flags; + unsigned has_scheme:1; h2_error error; enum vhd_ret_e vhd_ret; char *out; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 472b86294..daf4b93c5 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -127,7 +127,8 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } static h2_error -h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) +h2h_addhdr(struct h2h_decode *d, struct http *hp, char *b, size_t namelen, + size_t len) { /* XXX: This might belong in cache/cache_http.c */ const char *b0; @@ -188,7 +189,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ - if (*flags & H2H_DECODE_FLAG_SCHEME_SEEN) { + if (d->has_scheme) { VSLb(hp->vsl, SLT_BogoHeader, "Duplicate pseudo-header %.*s%.*s", (int)namelen, b0, @@ -199,7 +200,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) b++; len-=1; n = hp->nhd; - *flags |= H2H_DECODE_FLAG_SCHEME_SEEN; + d->has_scheme = 1; for (p = b + namelen, u = 0; u < len-namelen; p++, u++) { @@ -302,6 +303,9 @@ h2h_decode_fini(const struct h2_sess *h2) VSLb(h2->new_req->http->vsl, SLT_BogoHeader, "HPACK compression error/fini (%s)", VHD_Error(d->vhd_ret)); ret = H2CE_COMPRESSION_ERROR; + } else if (d->error == NULL && !d->has_scheme) { + VSLb(h2->vsl, SLT_Debug, "Missing :scheme"); + ret = H2SE_MISSING_SCHEME; //rfc7540,l,3087,3090 } else ret = d->error; FINI_OBJ(d); @@ -389,8 +393,8 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out_u); if (d->error) break; - d->error = h2h_addhdr(hp, d->out, d->namelen, d->out_u, - &d->flags); + d->error = h2h_addhdr(d, hp, d->out, + d->namelen, d->out_u); if (d->error) break; d->out[d->out_u++] = '\0'; /* Zero guard */ diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 593abffbb..aaa438778 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -627,13 +627,11 @@ static h2_error h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { - int scheme_seen; h2_error h2e; ssize_t cl; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); - scheme_seen = h2->decode->flags & H2H_DECODE_FLAG_SCHEME_SEEN; h2e = h2h_decode_fini(h2); h2->new_req = NULL; if (h2e != NULL) { @@ -690,11 +688,6 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 } - if (!(scheme_seen)) { - VSLb(h2->vsl, SLT_Debug, "Missing :scheme"); - return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 - } - AN(req->http->hd[HTTP_HDR_PROTO].b); if (*req->http->hd[HTTP_HDR_URL].b == '*' && diff --git a/include/tbl/h2_error.h b/include/tbl/h2_error.h index ed72f6b00..f4dc5ca2c 100644 --- a/include/tbl/h2_error.h +++ b/include/tbl/h2_error.h @@ -171,6 +171,15 @@ H2_ERROR( /* descr */ "http/2 rapid reset detected" ) +H2_ERROR( + /* name */ MISSING_SCHEME, + /* val */ 1, /* PROTOCOL_ERROR */ + /* types */ 2, + /* goaway */ 1, + /* reason */ SC_NULL, + /* descr */ "Missing :scheme pseudo-header" +) + H2_ERROR( /* name */ BROKE_WINDOW, /* val */ 8, /* CANCEL */ From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:09 +0000 (UTC) Subject: [7.4] b04725b64 vtc_http2: Print headers as "name: value" Message-ID: <20240404143009.E11521020C2@lists.varnish-cache.org> commit b04725b648ce1167a54c8bfcaa005e7fb0a3474c Author: Dridi Boukelmoune Date: Wed Mar 27 11:54:43 2024 +0100 vtc_http2: Print headers as "name: value" The extra space before the colon looked uncanny. The rest is just code indentation improvements. Better diff with the --ignore-all-space --word-diff options. diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 93c7d6508..1d401ed17 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -467,22 +467,20 @@ decode_hdr(struct http *hp, struct hpk_hdr *h, const struct vsb *vsb) r = HPK_DecHdr(iter, h + n); if (r == hpk_err ) break; - vtc_log(hp->vl, 4, - "header[%2d]: %s : %s", - n, - h[n].key.ptr, - h[n].value.ptr); + vtc_log(hp->vl, 4, "header[%2d]: %s: %s", + n, h[n].key.ptr, h[n].value.ptr); n++; if (r == hpk_done) break; } - if (r != hpk_done) + if (r != hpk_done) { vtc_log(hp->vl, hp->fatal ? 4 : 0, - "Header decoding failed (%d) %d", r, hp->fatal); - else if (n == MAX_HDR) + "Header decoding failed (%d) %d", r, hp->fatal); + } else if (n == MAX_HDR) { vtc_log(hp->vl, hp->fatal, - "Max number of headers reached (%d)", MAX_HDR); + "Max number of headers reached (%d)", MAX_HDR); + } HPK_FreeIter(iter); } From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:10 +0000 (UTC) Subject: [7.4] c5f616ce2 txt: New macros to work with strings Message-ID: <20240404143010.0A5061020C9@lists.varnish-cache.org> commit c5f616ce207d01b7a5432042ae99f07af47c695e Author: Dridi Boukelmoune Date: Wed Mar 27 15:19:01 2024 +0100 txt: New macros to work with strings diff --git a/include/vdef.h b/include/vdef.h index 2df601119..a321407d7 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -263,6 +263,8 @@ typedef struct { #define Tcheck(t) do { (void)pdiff((t).b, (t).e); } while (0) #define Tlen(t) (pdiff((t).b, (t).e)) +#define Tstr(s) ((txt){(s), (s) + strlen(s)}) +#define Tstrcmp(t, s) (strncmp((t).b, (s), Tlen(t))) /* #3020 dummy definitions until PR is merged*/ #define LIKELY(x) (x) From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:10 +0000 (UTC) Subject: [7.4] 9a1f08b00 http2_hpack: Reorganize header addition for clarity Message-ID: <20240404143010.61D181020D2@lists.varnish-cache.org> commit 9a1f08b000e6d902913ef233542779bb623ff761 Author: Dridi Boukelmoune Date: Wed Mar 27 16:09:19 2024 +0100 http2_hpack: Reorganize header addition for clarity Instead of passing both a decoder and individual decoder fields, the signature for h2h_addhdr() changed to only take the decoder. The order of parameters is destination first, then the source following the calling conventon of functions like memcpy(). Internally the function is reorganized with a bunch of txt variables to keep track of the header being added, its name and value. In addition to clarity, this also helps improve safety and correctness. For example the :authority pseudo-header name is erased in place to turn it into a regular host header, but having a dedicated txt for the header name allows its preservation. diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index daf4b93c5..48e6e5c5c 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -39,6 +39,18 @@ #include "http2/cache_http2.h" #include "vct.h" +static void +h2h_assert_ready(struct h2h_decode *d) +{ + + CHECK_OBJ_NOTNULL(d, H2H_DECODE_MAGIC); + AN(d->out); + assert(d->namelen >= 2); /* 2 chars from the ": " that we added */ + assert(d->namelen <= d->out_u); + assert(d->out[d->namelen - 2] == ':'); + assert(d->out[d->namelen - 1] == ' '); +} + // rfc9113,l,2493,2528 static h2_error h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) @@ -127,132 +139,130 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } static h2_error -h2h_addhdr(struct h2h_decode *d, struct http *hp, char *b, size_t namelen, - size_t len) +h2h_addhdr(struct http *hp, struct h2h_decode *d) { /* XXX: This might belong in cache/cache_http.c */ - const char *b0; + txt hdr, nm, val; int disallow_empty; + const char *p; unsigned n; - char *p; - unsigned u; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - AN(b); - assert(namelen >= 2); /* 2 chars from the ': ' that we added */ - assert(namelen <= len); + h2h_assert_ready(d); + + /* Assume hdr is by default a regular header from what we decoded. */ + hdr.b = d->out; + hdr.e = hdr.b + d->out_u; + n = hp->nhd; + + /* nm and val are separated by ": " */ + nm.b = hdr.b; + nm.e = nm.b + d->namelen - 2; + val.b = nm.e + 2; + val.e = hdr.e; disallow_empty = 0; - if (len > UINT_MAX) { /* XXX: cache_param max header size */ - VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", b); + if (Tlen(hdr) > UINT_MAX) { /* XXX: cache_param max header size */ + VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", hdr.b); return (H2SE_ENHANCE_YOUR_CALM); } - b0 = b; - if (b[0] == ':') { + if (*nm.b == ':') { /* Match H/2 pseudo headers */ /* XXX: Should probably have some include tbl for pseudo-headers */ - if (!strncmp(b, ":method: ", namelen)) { - b += namelen; - len -= namelen; + if (!Tstrcmp(nm, ":method")) { + hdr.b = val.b; n = HTTP_HDR_METHOD; disallow_empty = 1; - /* First field cannot contain SP or CTL */ - for (p = b, u = 0; u < len; p++, u++) { - if (vct_issp(*p) || vct_isctl(*p)) + /* Check HTTP token */ + for (p = hdr.b; p < hdr.e; p++) { + if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); } - } else if (!strncmp(b, ":path: ", namelen)) { - b += namelen; - len -= namelen; + } else if (!Tstrcmp(nm, ":path")) { + hdr.b = val.b; n = HTTP_HDR_URL; disallow_empty = 1; // rfc9113,l,2693,2705 - if (len > 0 && *b != '/' && - strncmp(b, "*", len) != 0) { + if (Tlen(val) > 0 && *val.b != '/' && + Tstrcmp(val, "*")) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal :path pseudo-header %.*s", - (int)len, b); + (int)Tlen(val), val.b); return (H2SE_PROTOCOL_ERROR); } - /* Second field cannot contain LWS or CTL */ - for (p = b, u = 0; u < len; p++, u++) { + /* Path cannot contain LWS or CTL */ + for (p = hdr.b; p < hdr.e; p++) { if (vct_islws(*p) || vct_isctl(*p)) return (H2SE_PROTOCOL_ERROR); } - } else if (!strncmp(b, ":scheme: ", namelen)) { + } else if (!Tstrcmp(nm, ":scheme")) { /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ if (d->has_scheme) { VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header %.*s%.*s", - (int)namelen, b0, - (int)(len > 20 ? 20 : len), b); + "Duplicate pseudo-header :scheme: %.*s", + vmin_t(int, Tlen(val), 20), val.b); return (H2SE_PROTOCOL_ERROR); } - b++; - len-=1; - n = hp->nhd; + hdr.b++; d->has_scheme = 1; + disallow_empty = 1; - for (p = b + namelen, u = 0; u < len-namelen; - p++, u++) { - if (vct_issp(*p) || vct_isctl(*p)) + /* Check HTTP token */ + for (p = val.b; p < val.e; p++) { + if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); } - - if (!u) - return (H2SE_PROTOCOL_ERROR); - } else if (!strncmp(b, ":authority: ", namelen)) { - b+=6; - len-=6; - memcpy(b, "host", 4); - n = hp->nhd; + } else if (!Tstrcmp(nm, ":authority")) { + /* NB: we inject "host" in place of "rity" for + * the ":authority" pseudo-header. + */ + memcpy(d->out + 6, "host", 4); + hdr.b += 6; + nm = Tstr(":authority"); /* preserve original */ } else { /* Unknown pseudo-header */ VSLb(hp->vsl, SLT_BogoHeader, "Unknown pseudo-header: %.*s", - (int)(len > 20 ? 20 : len), b); + vmin_t(int, Tlen(hdr), 20), hdr.b); return (H2SE_PROTOCOL_ERROR); // rfc7540,l,2990,2992 } - } else - n = hp->nhd; + } + + if (disallow_empty && Tlen(val) == 0) { + VSLb(hp->vsl, SLT_BogoHeader, + "Empty pseudo-header %.*s", + (int)Tlen(nm), nm.b); + return (H2SE_PROTOCOL_ERROR); + } if (n < HTTP_HDR_FIRST) { - /* Check for duplicate pseudo-header */ if (hp->hd[n].b != NULL) { VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header %.*s%.*s", - (int)namelen, b0, (int)(len > 20 ? 20 : len), b); + "Duplicate pseudo-header %.*s", + (int)Tlen(nm), nm.b); return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 } } else { /* Check for space in struct http */ if (n >= hp->shd) { - VSLb(hp->vsl, SLT_LostHeader, "Too many headers: %.*s", - (int)(len > 20 ? 20 : len), b); + VSLb(hp->vsl, SLT_LostHeader, + "Too many headers: %.*s", + vmin_t(int, Tlen(hdr), 20), hdr.b); return (H2SE_ENHANCE_YOUR_CALM); } hp->nhd++; } - hp->hd[n].b = b; - hp->hd[n].e = b + len; - - if (disallow_empty && !Tlen(hp->hd[n])) { - VSLb(hp->vsl, SLT_BogoHeader, - "Empty pseudo-header %.*s", - (int)namelen, b0); - return (H2SE_PROTOCOL_ERROR); - } - + hp->hd[n] = hdr; return (0); } @@ -393,8 +403,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out_u); if (d->error) break; - d->error = h2h_addhdr(d, hp, d->out, - d->namelen, d->out_u); + d->error = h2h_addhdr(hp, d); if (d->error) break; d->out[d->out_u++] = '\0'; /* Zero guard */ From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:10 +0000 (UTC) Subject: [7.4] 9abc57fe0 http2_hpack: Refuse multiple :authority pseudo headers Message-ID: <20240404143010.7FECE1020DA@lists.varnish-cache.org> commit 9abc57fe04ad0b1be06d92f7d1d59ece8e499a62 Author: Dridi Boukelmoune Date: Wed Mar 27 16:17:08 2024 +0100 http2_hpack: Refuse multiple :authority pseudo headers It became explicit in rfc9113: > The same pseudo-header field name MUST NOT appear more than once in a > field block. While at it, the duplicate pseudo-header error can be consolidated in a single location instead of adding one more branch. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 126a70d3a..5962ddb51 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -227,6 +227,7 @@ struct h2h_decode { unsigned magic; #define H2H_DECODE_MAGIC 0xd092bde4 + unsigned has_authority:1; unsigned has_scheme:1; h2_error error; enum vhd_ret_e vhd_ret; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 48e6e5c5c..a134c379a 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -145,7 +145,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) txt hdr, nm, val; int disallow_empty; const char *p; - unsigned n; + unsigned n, has_dup; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); h2h_assert_ready(d); @@ -162,6 +162,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) val.e = hdr.e; disallow_empty = 0; + has_dup = 0; if (Tlen(hdr) > UINT_MAX) { /* XXX: cache_param max header size */ VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", hdr.b); @@ -205,14 +206,8 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ - if (d->has_scheme) { - VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header :scheme: %.*s", - vmin_t(int, Tlen(val), 20), val.b); - return (H2SE_PROTOCOL_ERROR); - } - hdr.b++; + has_dup = d->has_scheme; d->has_scheme = 1; disallow_empty = 1; @@ -228,6 +223,8 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) memcpy(d->out + 6, "host", 4); hdr.b += 6; nm = Tstr(":authority"); /* preserve original */ + has_dup = d->has_authority; + d->has_authority = 1; } else { /* Unknown pseudo-header */ VSLb(hp->vsl, SLT_BogoHeader, @@ -244,14 +241,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) return (H2SE_PROTOCOL_ERROR); } - if (n < HTTP_HDR_FIRST) { - if (hp->hd[n].b != NULL) { - VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header %.*s", - (int)Tlen(nm), nm.b); - return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 - } - } else { + if (n >= HTTP_HDR_FIRST) { /* Check for space in struct http */ if (n >= hp->shd) { VSLb(hp->vsl, SLT_LostHeader, @@ -260,6 +250,15 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) return (H2SE_ENHANCE_YOUR_CALM); } hp->nhd++; + AZ(hp->hd[n].b); + } + + if (has_dup || hp->hd[n].b != NULL) { + assert(nm.b[0] == ':'); + VSLb(hp->vsl, SLT_BogoHeader, + "Duplicate pseudo-header %.*s", + (int)Tlen(nm), nm.b); + return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 } hp->hd[n] = hdr; diff --git a/bin/varnishtest/tests/r02351.vtc b/bin/varnishtest/tests/r02351.vtc index 09d12e541..936f522b9 100644 --- a/bin/varnishtest/tests/r02351.vtc +++ b/bin/varnishtest/tests/r02351.vtc @@ -1,4 +1,4 @@ -varnishtest "#2351: :path/:method error handling" +varnishtest "#2351: h2 pseudo-headers error handling" server s1 { rxreq @@ -43,6 +43,16 @@ client c1 { } -run } -run +client c2 { + # Duplicate :authority + stream next { + txreq -noadd -hdr :path / -hdr :method GET -hdr :scheme http \ + -hdr :authority example.com -hdr :authority example.org + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + varnish v1 -expect MEMPOOL.req0.live == 0 varnish v1 -expect MEMPOOL.req1.live == 0 varnish v1 -expect MEMPOOL.sess0.live == 0 From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:10 +0000 (UTC) Subject: [7.4] f1b79d9b4 http2_hpack: Remove one level of nesting Message-ID: <20240404143010.A2E7F1020E6@lists.varnish-cache.org> commit f1b79d9b4c8eac75cebce3daf21d141e42b2cc44 Author: Dridi Boukelmoune Date: Wed Mar 27 16:28:18 2024 +0100 http2_hpack: Remove one level of nesting Better diff with the --ignore-all-space option. diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index a134c379a..c2013c1f2 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -169,69 +169,64 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) return (H2SE_ENHANCE_YOUR_CALM); } - if (*nm.b == ':') { - /* Match H/2 pseudo headers */ - /* XXX: Should probably have some include tbl for - pseudo-headers */ - if (!Tstrcmp(nm, ":method")) { - hdr.b = val.b; - n = HTTP_HDR_METHOD; - disallow_empty = 1; - - /* Check HTTP token */ - for (p = hdr.b; p < hdr.e; p++) { - if (!vct_istchar(*p)) - return (H2SE_PROTOCOL_ERROR); - } - } else if (!Tstrcmp(nm, ":path")) { - hdr.b = val.b; - n = HTTP_HDR_URL; - disallow_empty = 1; - - // rfc9113,l,2693,2705 - if (Tlen(val) > 0 && *val.b != '/' && - Tstrcmp(val, "*")) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal :path pseudo-header %.*s", - (int)Tlen(val), val.b); + /* Match H/2 pseudo headers */ + /* XXX: Should probably have some include tbl for pseudo-headers */ + if (!Tstrcmp(nm, ":method")) { + hdr.b = val.b; + n = HTTP_HDR_METHOD; + disallow_empty = 1; + + /* Check HTTP token */ + for (p = hdr.b; p < hdr.e; p++) { + if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); - } + } + } else if (!Tstrcmp(nm, ":path")) { + hdr.b = val.b; + n = HTTP_HDR_URL; + disallow_empty = 1; - /* Path cannot contain LWS or CTL */ - for (p = hdr.b; p < hdr.e; p++) { - if (vct_islws(*p) || vct_isctl(*p)) - return (H2SE_PROTOCOL_ERROR); - } - } else if (!Tstrcmp(nm, ":scheme")) { - /* XXX: What to do about this one? (typically - "http" or "https"). For now set it as a normal - header, stripping the first ':'. */ - hdr.b++; - has_dup = d->has_scheme; - d->has_scheme = 1; - disallow_empty = 1; - - /* Check HTTP token */ - for (p = val.b; p < val.e; p++) { - if (!vct_istchar(*p)) - return (H2SE_PROTOCOL_ERROR); - } - } else if (!Tstrcmp(nm, ":authority")) { - /* NB: we inject "host" in place of "rity" for - * the ":authority" pseudo-header. - */ - memcpy(d->out + 6, "host", 4); - hdr.b += 6; - nm = Tstr(":authority"); /* preserve original */ - has_dup = d->has_authority; - d->has_authority = 1; - } else { - /* Unknown pseudo-header */ + // rfc9113,l,2693,2705 + if (Tlen(val) > 0 && val.b[0] != '/' && Tstrcmp(val, "*")) { VSLb(hp->vsl, SLT_BogoHeader, - "Unknown pseudo-header: %.*s", - vmin_t(int, Tlen(hdr), 20), hdr.b); - return (H2SE_PROTOCOL_ERROR); // rfc7540,l,2990,2992 + "Illegal :path pseudo-header %.*s", + (int)Tlen(val), val.b); + return (H2SE_PROTOCOL_ERROR); } + + /* Path cannot contain LWS or CTL */ + for (p = hdr.b; p < hdr.e; p++) { + if (vct_islws(*p) || vct_isctl(*p)) + return (H2SE_PROTOCOL_ERROR); + } + } else if (!Tstrcmp(nm, ":scheme")) { + /* XXX: What to do about this one? (typically + "http" or "https"). For now set it as a normal + header, stripping the first ':'. */ + hdr.b++; + has_dup = d->has_scheme; + d->has_scheme = 1; + disallow_empty = 1; + + /* Check HTTP token */ + for (p = val.b; p < val.e; p++) { + if (!vct_istchar(*p)) + return (H2SE_PROTOCOL_ERROR); + } + } else if (!Tstrcmp(nm, ":authority")) { + /* NB: we inject "host" in place of "rity" for + * the ":authority" pseudo-header. + */ + memcpy(d->out + 6, "host", 4); + hdr.b += 6; + nm = Tstr(":authority"); /* preserve original */ + has_dup = d->has_authority; + d->has_authority = 1; + } else if (nm.b[0] == ':') { + VSLb(hp->vsl, SLT_BogoHeader, + "Unknown pseudo-header: %.*s", + vmin_t(int, Tlen(hdr), 20), hdr.b); + return (H2SE_PROTOCOL_ERROR); // rfc7540,l,2990,2992 } if (disallow_empty && Tlen(val) == 0) { From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:10 +0000 (UTC) Subject: [7.4] 0893bd9e2 http2_hpack: Also rewrite h2h_checkhdr() for clarity Message-ID: <20240404143010.CDC351020F5@lists.varnish-cache.org> commit 0893bd9e27b77ce3aef1440265df24a199fdfb04 Author: Dridi Boukelmoune Date: Thu Mar 28 14:13:36 2024 +0100 http2_hpack: Also rewrite h2h_checkhdr() for clarity It does a first pass on header names and values, and only logs errors, so the signature is updated accordingly and the call site is moved into h2h_addhdr(). diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index c2013c1f2..531c50307 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -53,9 +53,10 @@ h2h_assert_ready(struct h2h_decode *d) // rfc9113,l,2493,2528 static h2_error -h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) +h2h_checkhdr(struct vsl_log *vsl, txt nm, txt val) { const char *p; + int l; enum { FLD_NAME_FIRST, FLD_NAME, @@ -63,23 +64,17 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) FLD_VALUE } state; - CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - AN(b); - assert(namelen >= 2); /* 2 chars from the ': ' that we added */ - assert(namelen <= len); - assert(b[namelen - 2] == ':'); - assert(b[namelen - 1] == ' '); - - if (namelen == 2) { - VSLb(hp->vsl, SLT_BogoHeader, "Empty name"); + if (Tlen(nm) == 0) { + VSLb(vsl, SLT_BogoHeader, "Empty name"); return (H2SE_PROTOCOL_ERROR); } - // VSLb(hp->vsl, SLT_Debug, "CHDR [%.*s] [%.*s]", - // (int)namelen, b, (int)(len - namelen), b + namelen); + // VSLb(vsl, SLT_Debug, "CHDR [%.*s] [%.*s]", + // (int)Tlen(nm), nm.b, (int)Tlen(val), val.b); + l = vmin_t(int, Tlen(nm) + 2 + Tlen(val), 20); state = FLD_NAME_FIRST; - for (p = b; p < b + namelen - 2; p++) { + for (p = nm.b; p < nm.e; p++) { switch(state) { case FLD_NAME_FIRST: state = FLD_NAME; @@ -88,15 +83,15 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) /* FALL_THROUGH */ case FLD_NAME: if (isupper(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, + VSLb(vsl, SLT_BogoHeader, "Illegal field header name (upper-case): %.*s", - (int)(len > 20 ? 20 : len), b); + l, nm.b); return (H2SE_PROTOCOL_ERROR); } if (!vct_istchar(*p) || *p == ':') { - VSLb(hp->vsl, SLT_BogoHeader, + VSLb(vsl, SLT_BogoHeader, "Illegal field header name (non-token): %.*s", - (int)(len > 20 ? 20 : len), b); + l, nm.b); return (H2SE_PROTOCOL_ERROR); } break; @@ -106,22 +101,20 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } state = FLD_VALUE_FIRST; - for (p = b + namelen; p < b + len; p++) { + for (p = val.b; p < val.e; p++) { switch(state) { case FLD_VALUE_FIRST: if (vct_issp(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field value start %.*s", - (int)(len > 20 ? 20 : len), b); + VSLb(vsl, SLT_BogoHeader, + "Illegal field value start %.*s", l, nm.b); return (H2SE_PROTOCOL_ERROR); } state = FLD_VALUE; /* FALL_THROUGH */ case FLD_VALUE: if (!vct_ishdrval(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field value %.*s", - (int)(len > 20 ? 20 : len), b); + VSLb(vsl, SLT_BogoHeader, + "Illegal field value %.*s", l, nm.b); return (H2SE_PROTOCOL_ERROR); } break; @@ -129,10 +122,9 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) WRONG("http2 field value validation state"); } } - if (state == FLD_VALUE && vct_issp(b[len - 1])) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field value (end) %.*s", - (int)(len > 20 ? 20 : len), b); + if (state == FLD_VALUE && vct_issp(val.e[-1])) { + VSLb(vsl, SLT_BogoHeader, + "Illegal field value (end) %.*s", l, nm.b); return (H2SE_PROTOCOL_ERROR); } return (0); @@ -146,6 +138,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) int disallow_empty; const char *p; unsigned n, has_dup; + h2_error err; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); h2h_assert_ready(d); @@ -161,6 +154,10 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) val.b = nm.e + 2; val.e = hdr.e; + err = h2h_checkhdr(hp->vsl, nm, val); + if (err != NULL) + return (err); + disallow_empty = 0; has_dup = 0; @@ -393,10 +390,6 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->error = H2SE_ENHANCE_YOUR_CALM; break; } - d->error = h2h_checkhdr(hp, d->out, d->namelen, - d->out_u); - if (d->error) - break; d->error = h2h_addhdr(hp, d); if (d->error) break; From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:10 +0000 (UTC) Subject: [7.4] 54e2462bd http2_hpack: Enforce http_req_hdr_len limit Message-ID: <20240404143011.0C8E6102103@lists.varnish-cache.org> commit 54e2462bd9bb12bddc86822aff38f4788bd3f009 Author: Dridi Boukelmoune Date: Thu Mar 28 15:21:01 2024 +0100 http2_hpack: Enforce http_req_hdr_len limit Refs #3709 diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 531c50307..b47dba507 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -161,7 +161,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) disallow_empty = 0; has_dup = 0; - if (Tlen(hdr) > UINT_MAX) { /* XXX: cache_param max header size */ + if (Tlen(hdr) > cache_param->http_req_hdr_len) { VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", hdr.b); return (H2SE_ENHANCE_YOUR_CALM); } diff --git a/bin/varnishtest/tests/r03709.vtc b/bin/varnishtest/tests/r03709.vtc new file mode 100644 index 000000000..7439efba3 --- /dev/null +++ b/bin/varnishtest/tests/r03709.vtc @@ -0,0 +1,21 @@ +varnishtest "h2 req limits" + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set http_req_hdr_len 40b" +varnish v1 -vcl { + backend be none; +} -start + +logexpect l1 -v v1 -g raw -q BogoHeader { + expect 0 1001 BogoHeader "Header too large: :path" +} -start + +client c1 { + stream next { + txreq -url ${string,repeat,4,/123456789} + rxrst + expect rst.err == ENHANCE_YOUR_CALM + } -run +} -run + +logexpect l1 -wait From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:11 +0000 (UTC) Subject: [7.4] 698ba0500 http2: New H2_ERROR_MATCH() helper macro Message-ID: <20240404143011.30A28102112@lists.varnish-cache.org> commit 698ba0500f06e22229c706e63d231d379ffa32fc Author: Dridi Boukelmoune Date: Thu Mar 28 15:56:21 2024 +0100 http2: New H2_ERROR_MATCH() helper macro diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 5962ddb51..9fbb58443 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -50,6 +50,9 @@ struct h2_error_s { typedef const struct h2_error_s *h2_error; +#define H2_ERROR_MATCH(err, target) \ + ((err) != NULL && (err)->val == (target)->val) + #define H2_CUSTOM_ERRORS #define H2EC1(U,v,g,r,d) extern const struct h2_error_s H2CE_##U[1]; #define H2EC2(U,v,g,r,d) extern const struct h2_error_s H2SE_##U[1]; diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index d642e31df..d1418757c 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -447,6 +447,6 @@ H2_Send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, h2_send(wrk, r2, ftyp, flags, len, ptr, counter); h2e = h2_errcheck(r2, r2->h2sess); - if (h2e != NULL && h2e->val == H2SE_CANCEL->val) + if (H2_ERROR_MATCH(h2e, H2SE_CANCEL)) H2_Send_RST(wrk, r2->h2sess, r2, r2->stream, h2e); } From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:11 +0000 (UTC) Subject: [7.4] f7413ffb5 http2_hpack: New custom h2 error for http_req_size Message-ID: <20240404143011.6639310211F@lists.varnish-cache.org> commit f7413ffb57c2bb11fdd49815e69e44eea9ea1f7c Author: Dridi Boukelmoune Date: Thu Mar 28 16:02:53 2024 +0100 http2_hpack: New custom h2 error for http_req_size diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index b47dba507..92858543e 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -343,7 +343,8 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) /* Only H2E_ENHANCE_YOUR_CALM indicates that we should continue processing. Other errors should have been returned and handled by the caller. */ - assert(d->error == 0 || d->error == H2SE_ENHANCE_YOUR_CALM); + if (d->error != NULL) + assert(H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)); while (1) { AN(d->out); @@ -362,7 +363,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; } - if (d->error == H2SE_ENHANCE_YOUR_CALM) { + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { d->out_u = 0; assert(d->out_u < d->out_l); continue; @@ -374,7 +375,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) case VHD_NAME: assert(d->namelen == 0); if (d->out_l - d->out_u < 2) { - d->error = H2SE_ENHANCE_YOUR_CALM; + d->error = H2SE_REQ_SIZE; break; } d->out[d->out_u++] = ':'; @@ -387,7 +388,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) case VHD_VALUE: assert(d->namelen > 0); if (d->out_l - d->out_u < 1) { - d->error = H2SE_ENHANCE_YOUR_CALM; + d->error = H2SE_REQ_SIZE; break; } d->error = h2h_addhdr(hp, d); @@ -401,7 +402,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; case VHD_BUF: - d->error = H2SE_ENHANCE_YOUR_CALM; + d->error = H2SE_REQ_SIZE; break; default: @@ -409,7 +410,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; } - if (d->error == H2SE_ENHANCE_YOUR_CALM) { + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { d->out = d->reset; d->out_l = e - d->out; d->out_u = 0; @@ -418,7 +419,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; } - if (d->error == H2SE_ENHANCE_YOUR_CALM) + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) return (0); /* Stream error, delay reporting until h2h_decode_fini so that we can process the complete header block */ diff --git a/include/tbl/h2_error.h b/include/tbl/h2_error.h index f4dc5ca2c..adfbbde42 100644 --- a/include/tbl/h2_error.h +++ b/include/tbl/h2_error.h @@ -197,6 +197,15 @@ H2_ERROR( /* reason */ SC_BANKRUPT, /* descr */ "http/2 bankrupt connection" ) + +H2_ERROR( + /* name */ REQ_SIZE, + /* val */ 11, /* ENHANCE_YOUR_CALM */ + /* types */ 2, + /* goaway */ 0, + /* reason */ SC_NULL, + /* descr */ "HTTP/2 header list exceeded http_req_size" +) # undef H2_CUSTOM_ERRORS #endif From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:11 +0000 (UTC) Subject: [7.4] b96d83d0a http2_hpack: Enforce http_req_size limit Message-ID: <20240404143011.9B43810212E@lists.varnish-cache.org> commit b96d83d0ad01ede2304dc9e7c8884d3ba92227e6 Author: Dridi Boukelmoune Date: Thu Mar 28 16:08:46 2024 +0100 http2_hpack: Enforce http_req_size limit Fixes #3709 Closes #3892 diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 92858543e..4d155e3d1 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -269,7 +269,8 @@ h2h_decode_init(const struct h2_sess *h2) d = h2->decode; INIT_OBJ(d, H2H_DECODE_MAGIC); VHD_Init(d->vhd); - d->out_l = WS_ReserveAll(h2->new_req->http->ws); + d->out_l = WS_ReserveSize(h2->new_req->http->ws, + cache_param->http_req_size); /* * Can't do any work without any buffer * space. Require non-zero size. @@ -310,6 +311,10 @@ h2h_decode_fini(const struct h2_sess *h2) } else ret = d->error; FINI_OBJ(d); + if (ret == H2SE_REQ_SIZE) { + VSLb(h2->new_req->http->vsl, SLT_LostHeader, + "Header list too large"); + } return (ret); } diff --git a/bin/varnishtest/tests/r03709.vtc b/bin/varnishtest/tests/r03709.vtc index 7439efba3..242afe2f1 100644 --- a/bin/varnishtest/tests/r03709.vtc +++ b/bin/varnishtest/tests/r03709.vtc @@ -2,17 +2,40 @@ varnishtest "h2 req limits" varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set http_req_hdr_len 40b" +varnish v1 -cliok "param.set http_req_size 512b" varnish v1 -vcl { backend be none; } -start -logexpect l1 -v v1 -g raw -q BogoHeader { +logexpect l1 -v v1 -g raw -q BogoHeader,LostHeader { expect 0 1001 BogoHeader "Header too large: :path" + expect 0 1002 LostHeader "Header list too large" } -start client c1 { stream next { - txreq -url ${string,repeat,4,/123456789} + txreq -url ${string,repeat,4,/123456789} \ + -hdr limit http_req_hdr_len + rxrst + expect rst.err == ENHANCE_YOUR_CALM + } -run + + stream next { + txreq -url "/http_req_size" \ + -hdr hdr1 ${string,repeat,3,/123456789} \ + -hdr hdr2 ${string,repeat,3,/123456789} \ + -hdr hdr3 ${string,repeat,3,/123456789} \ + -hdr hdr4 ${string,repeat,3,/123456789} \ + -hdr hdr5 ${string,repeat,3,/123456789} \ + -hdr hdr6 ${string,repeat,3,/123456789} \ + -hdr hdr7 ${string,repeat,3,/123456789} \ + -hdr hdr8 ${string,repeat,3,/123456789} \ + -hdr hdr9 ${string,repeat,3,/123456789} \ + -hdr hdr10 ${string,repeat,3,/123456789} \ + -hdr hdr11 ${string,repeat,3,/123456789} \ + -hdr hdr12 ${string,repeat,3,/123456789} \ + -hdr hdr13 ${string,repeat,3,/123456789} \ + -hdr hdr14 ${string,repeat,3,/123456789} rxrst expect rst.err == ENHANCE_YOUR_CALM } -run From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:11 +0000 (UTC) Subject: [7.4] a94076503 vtc: Add http_max_hdr coverage to r3709 Message-ID: <20240404143011.BD7C7102132@lists.varnish-cache.org> commit a94076503fde227317e4b410834e9101768be240 Author: Dridi Boukelmoune Date: Thu Mar 28 16:22:59 2024 +0100 vtc: Add http_max_hdr coverage to r3709 For the sole purpose of having these limits tested in a single place. diff --git a/bin/varnishtest/tests/r03709.vtc b/bin/varnishtest/tests/r03709.vtc index 242afe2f1..65038fcce 100644 --- a/bin/varnishtest/tests/r03709.vtc +++ b/bin/varnishtest/tests/r03709.vtc @@ -3,6 +3,7 @@ varnishtest "h2 req limits" varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set http_req_hdr_len 40b" varnish v1 -cliok "param.set http_req_size 512b" +varnish v1 -cliok "param.set http_max_hdr 32" varnish v1 -vcl { backend be none; } -start @@ -10,6 +11,7 @@ varnish v1 -vcl { logexpect l1 -v v1 -g raw -q BogoHeader,LostHeader { expect 0 1001 BogoHeader "Header too large: :path" expect 0 1002 LostHeader "Header list too large" + expect 0 1003 LostHeader "Too many headers" } -start client c1 { @@ -39,6 +41,41 @@ client c1 { rxrst expect rst.err == ENHANCE_YOUR_CALM } -run + + stream next { + txreq -url "/http_max_hdr" \ + -hdr hdr1 val1 \ + -hdr hdr2 val2 \ + -hdr hdr3 val3 \ + -hdr hdr4 val4 \ + -hdr hdr4 val4 \ + -hdr hdr5 val5 \ + -hdr hdr6 val6 \ + -hdr hdr7 val7 \ + -hdr hdr8 val8 \ + -hdr hdr9 val9 \ + -hdr hdr10 val10 \ + -hdr hdr11 val11 \ + -hdr hdr11 val11 \ + -hdr hdr11 val11 \ + -hdr hdr12 val12 \ + -hdr hdr13 val13 \ + -hdr hdr14 val14 \ + -hdr hdr15 val15 \ + -hdr hdr16 val16 \ + -hdr hdr17 val17 \ + -hdr hdr18 val18 \ + -hdr hdr19 val19 \ + -hdr hdr20 val20 \ + -hdr hdr20 val20 \ + -hdr hdr21 val21 \ + -hdr hdr22 val22 \ + -hdr hdr23 val23 \ + -hdr hdr24 val24 \ + -hdr hdr25 val25 + rxrst + expect rst.err == ENHANCE_YOUR_CALM + } -run } -run logexpect l1 -wait From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:11 +0000 (UTC) Subject: [7.4] 7e47da722 param: Document mapping to HTTP/2 settings Message-ID: <20240404143011.EA83010213B@lists.varnish-cache.org> commit 7e47da7222d4fe84ebb89e992ca102624758b33d Author: Dridi Boukelmoune Date: Thu Mar 28 16:34:35 2024 +0100 param: Document mapping to HTTP/2 settings With the exception of h2_max_header_list_size that is not advertised as such despite being ent as part of the initial SETTINGS frame. The same parameter also sees its default and maximum values updated to 2^32-1. This is based on this sentence from rfc9113: > The initial value of this setting is unlimited. This aligns the h2_max_header_list_size parameter with the values set in h2_settings.h for MAX_HEADER_LIST_SIZE. diff --git a/include/tbl/params.h b/include/tbl/params.h index 600bf6a55..2a31810fb 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1207,6 +1207,12 @@ PARAM_SIMPLE( /* flags */ WIZARD ) +#define H2_SETTING_NAME(nm) "SETTINGS_" #nm +#define H2_SETTING_DESCR(nm) \ + "\n\nThe value of this parameter defines " H2_SETTING_NAME(nm) \ + " in the initial SETTINGS frame sent to the client when a new " \ + "HTTP2 session is established." + PARAM_SIMPLE( /* name */ h2_header_table_size, /* type */ bytes_u, @@ -1218,6 +1224,7 @@ PARAM_SIMPLE( "HTTP2 header table size.\n" "This is the size that will be used for the HPACK dynamic\n" "decoding table." + H2_SETTING_DESCR(HEADER_TABLE_SIZE) ) PARAM_SIMPLE( @@ -1231,6 +1238,7 @@ PARAM_SIMPLE( "HTTP2 Maximum number of concurrent streams.\n" "This is the number of requests that can be active\n" "at the same time for a single HTTP2 connection." + H2_SETTING_DESCR(MAX_CONCURRENT_STREAMS) ) /* We have a strict min at the protocol default here. This is because we @@ -1246,7 +1254,8 @@ PARAM_SIMPLE( /* def */ "65535b", /* units */ "bytes", /* descr */ - "HTTP2 initial flow control window size.", + "HTTP2 initial flow control window size." + H2_SETTING_DESCR(INITIAL_WINDOW_SIZE) ) PARAM_SIMPLE( @@ -1258,19 +1267,23 @@ PARAM_SIMPLE( /* units */ "bytes", /* descr */ "HTTP2 maximum per frame payload size we are willing to accept." + H2_SETTING_DESCR(MAX_FRAME_SIZE) ) PARAM_SIMPLE( /* name */ h2_max_header_list_size, /* type */ bytes_u, /* min */ "0b", - /* max */ NULL, - /* def */ "2147483647b", + /* max */ "4294967295b", + /* def */ "4294967295b", /* units */ "bytes", /* descr */ "HTTP2 maximum size of an uncompressed header list." ) +#undef H2_SETTING_DESCR +#undef H2_SETTING_NAME + #define H2_RR_INFO \ "Changes to this parameter affect the default for new HTTP2 " \ "sessions. vmod_h2(3) can be used to adjust it from VCL." From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:12 +0000 (UTC) Subject: [7.4] 39ca1d9ad http2_session: Advertise http_req_size to clients Message-ID: <20240404143012.3027210214D@lists.varnish-cache.org> commit 39ca1d9adda2bea28247b419a32e4ec9fd382fe4 Author: Dridi Boukelmoune Date: Thu Mar 28 16:35:52 2024 +0100 http2_session: Advertise http_req_size to clients Since http_req_size was already established for this purpose, and is now enforced for h2 traffic, it should naturally become the basis for the MAX_HEADER_LIST_SIZE setting in the initial SETTINGS frame sent to clients. The h2_max_header_list_size parameter will grow a new purpose. diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index acbfdbb51..605798810 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -85,6 +85,7 @@ h2_local_settings(struct h2_settings *h2s) h2s->l = cache_param->h2_##l; #include "tbl/h2_settings.h" #undef H2_SETTINGS_PARAM_ONLY + h2s->max_header_list_size = cache_param->http_req_size; } /********************************************************************** diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index a139cc33a..97120a9e7 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -69,7 +69,7 @@ varnish v1 -expect MEMPOOL.sess1.live == 0 process p1 -stop # shell {cat ${tmpdir}/vlog} # SETTINGS with default initial window size -shell -match {1001 H2TxHdr c \[000006040000000000\]} { +shell -match {1001 H2TxHdr c \[00000c040000000000\]} { cat ${tmpdir}/vlog } diff --git a/bin/varnishtest/tests/t02005.vtc b/bin/varnishtest/tests/t02005.vtc index c5176f725..39737f93a 100644 --- a/bin/varnishtest/tests/t02005.vtc +++ b/bin/varnishtest/tests/t02005.vtc @@ -32,7 +32,7 @@ varnish v1 -cliok "param.set debug +syncvsl" logexpect l1 -v v1 -g raw { expect * 1001 ReqAcct "80 7 87 78 8 86" - expect * 1000 ReqAcct "45 8 53 63 28 91" + expect * 1000 ReqAcct "45 8 53 63 34 97" } -start client c1 { diff --git a/include/tbl/h2_settings.h b/include/tbl/h2_settings.h index 6c4520711..2dbac671f 100644 --- a/include/tbl/h2_settings.h +++ b/include/tbl/h2_settings.h @@ -92,6 +92,7 @@ H2_SETTING( // rfc7540,l,2150,2157 H2CE_PROTOCOL_ERROR ) +#ifndef H2_SETTINGS_PARAM_ONLY H2_SETTING( // rfc7540,l,2159,2167 MAX_HEADER_LIST_SIZE, max_header_list_size, @@ -101,6 +102,7 @@ H2_SETTING( // rfc7540,l,2159,2167 0xffffffff, 0 ) +#endif #undef H2_SETTING /*lint -restore */ diff --git a/include/tbl/params.h b/include/tbl/params.h index 2a31810fb..687a7c2f2 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -605,11 +605,13 @@ PARAM_SIMPLE( /* units */ "bytes", /* descr */ "Maximum number of bytes of HTTP client request we will deal with. " - " This is a limit on all bytes up to the double blank line which " - "ends the HTTP request.\n" + "This is a limit on all bytes up to the double blank line which " + "ends the HTTP request. " "The memory for the request is allocated from the client workspace " "(param: workspace_client) and this parameter limits how much of " - "that the request is allowed to take up." + "that the request is allowed to take up.\n\n" + "For HTTP2 clients, it is advertised as MAX_HEADER_LIST_SIZE in " + "the initial SETTINGS frame." ) PARAM_SIMPLE( @@ -1278,7 +1280,9 @@ PARAM_SIMPLE( /* def */ "4294967295b", /* units */ "bytes", /* descr */ - "HTTP2 maximum size of an uncompressed header list." + "HTTP2 maximum size of an uncompressed header list. This parameter " + "is not mapped to " H2_SETTING_NAME(MAX_HEADER_LIST_SIZE) " in the " + "initial SETTINGS frame, the http_req_size parameter is instead." ) #undef H2_SETTING_DESCR From dridi.boukelmoune at gmail.com Thu Apr 4 14:30:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:30:12 +0000 (UTC) Subject: [7.4] 0c3086e45 http2_hpack: Enforce h2_max_header_list_size Message-ID: <20240404143012.56C90102159@lists.varnish-cache.org> commit 0c3086e458af8df0f8fb5068275aa5e3177fbddf Author: Dridi Boukelmoune Date: Thu Mar 28 16:50:39 2024 +0100 http2_hpack: Enforce h2_max_header_list_size This parameter has a new role that consists in interrupting connections when decoding an HPACK block leads to a header list so large that the client must be stopped. By default, too large is 150% of http_req_size. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 9fbb58443..89e32309c 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -236,6 +236,7 @@ struct h2h_decode { enum vhd_ret_e vhd_ret; char *out; char *reset; + int64_t limit; size_t out_l; size_t out_u; size_t namelen; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 4d155e3d1..c4274a656 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -278,6 +278,14 @@ h2h_decode_init(const struct h2_sess *h2) XXXAN(d->out_l); d->out = WS_Reservation(h2->new_req->http->ws); d->reset = d->out; + + if (cache_param->h2_max_header_list_size == 0) + d->limit = h2->local_settings.max_header_list_size * 1.5; + else + d->limit = cache_param->h2_max_header_list_size; + + if (d->limit < h2->local_settings.max_header_list_size) + d->limit = INT64_MAX; } /* Possible error returns: @@ -351,7 +359,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) if (d->error != NULL) assert(H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)); - while (1) { + while (d->limit >= 0) { AN(d->out); assert(d->out_u <= d->out_l); d->vhd_ret = VHD_Decode(d->vhd, h2->dectbl, in, in_l, &in_u, @@ -369,6 +377,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) } if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { + d->limit -= d->out_u; d->out_u = 0; assert(d->out_u < d->out_l); continue; @@ -402,6 +411,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out[d->out_u++] = '\0'; /* Zero guard */ d->out += d->out_u; d->out_l -= d->out_u; + d->limit -= d->out_u; d->out_u = 0; d->namelen = 0; break; @@ -418,15 +428,25 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { d->out = d->reset; d->out_l = e - d->out; + d->limit -= d->out_u; d->out_u = 0; assert(d->out_l > 0); } else if (d->error) break; } - if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) - return (0); /* Stream error, delay reporting until - h2h_decode_fini so that we can process the - complete header block */ + if (d->limit < 0) { + /* Fatal error, the client exceeded both http_req_size + * and h2_max_header_list_size. */ + VSLb(h2->vsl, SLT_SessError, "Header list too large"); + return (H2CE_ENHANCE_YOUR_CALM); + } + + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { + /* Stream error, delay reporting until h2h_decode_fini so + * that we can process the complete header block. */ + return (NULL); + } + return (d->error); } diff --git a/bin/varnishtest/tests/f00015.vtc b/bin/varnishtest/tests/f00015.vtc new file mode 100644 index 000000000..de5df8ed2 --- /dev/null +++ b/bin/varnishtest/tests/f00015.vtc @@ -0,0 +1,19 @@ +varnishtest "h2 CONTINUATION flood" + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set vsl_mask -H2RxHdr,-H2RxBody" +varnish v1 -vcl { backend be none; } -start + +client c1 { + non_fatal + stream next { + txreq -nohdrend + loop 1000 { + txcont -nohdrend -hdr noise ${string,repeat,4096,x} + } + txcont -hdr last header + } -run +} -run + +varnish v1 -expect MAIN.s_req_hdrbytes < 65536 +varnish v1 -expect MAIN.sc_overload == 1 diff --git a/include/tbl/params.h b/include/tbl/params.h index 687a7c2f2..4ffdddae1 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1277,12 +1277,20 @@ PARAM_SIMPLE( /* type */ bytes_u, /* min */ "0b", /* max */ "4294967295b", - /* def */ "4294967295b", + /* def */ "0b", /* units */ "bytes", /* descr */ "HTTP2 maximum size of an uncompressed header list. This parameter " "is not mapped to " H2_SETTING_NAME(MAX_HEADER_LIST_SIZE) " in the " - "initial SETTINGS frame, the http_req_size parameter is instead." + "initial SETTINGS frame, the http_req_size parameter is instead.\n\n" + "The http_req_size advises HTTP2 clients of the maximum size for " + "the header list. Exceeding http_req_size results in a reset stream " + "after processing the HPACK block to perserve the connection, but " + "exceeding h2_max_header_list_size results in the HTTP2 connection " + "going away immediately.\n\n" + "If h2_max_header_list_size is lower than http_req_size, it has no " + "effect, except for the special value zero interpreted as 150% of " + "http_req_size." ) #undef H2_SETTING_DESCR From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:07 +0000 (UTC) Subject: [6.0] c89cc48ac vav: VAV_ParseTxt() to work without a null terminator Message-ID: <20240404143307.5F831103835@lists.varnish-cache.org> commit c89cc48aca3ed35a484a6463100319455758f2c2 Author: Dridi Boukelmoune Date: Thu Jun 17 11:51:21 2021 +0200 vav: VAV_ParseTxt() to work without a null terminator I kept a variable called "s" to cut noise from the diff. diff --git a/include/vav.h b/include/vav.h index a3861cf40..37d4af1c7 100644 --- a/include/vav.h +++ b/include/vav.h @@ -29,6 +29,7 @@ */ void VAV_Free(char **argv); +char **VAV_ParseTxt(const char *b, const char *e, int *argc, int flag); char **VAV_Parse(const char *s, int *argc, int flag); char *VAV_BackSlashDecode(const char *s, const char *e); int VAV_BackSlash(const char *s, char *res); diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index f1bc3db10..e7150df0c 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -134,14 +134,17 @@ static char err_invalid_backslash[] = "Invalid backslash sequence"; static char err_missing_quote[] = "Missing '\"'"; char ** -VAV_Parse(const char *s, int *argc, int flag) +VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) { char **argv; - const char *p; + const char *s, *p; int nargv, largv; int i, quote; - assert(s != NULL); + AN(b); + if (e == NULL) + e = strchr(b, '\0'); + s = b; nargv = 1; largv = 16; argv = calloc(largv, sizeof *argv); @@ -149,7 +152,7 @@ VAV_Parse(const char *s, int *argc, int flag) return (NULL); for (;;) { - if (*s == '\0') + if (s >= e) break; if (isspace(*s)) { s++; @@ -175,7 +178,7 @@ VAV_Parse(const char *s, int *argc, int flag) continue; } if (!quote) { - if (*s == '\0' || isspace(*s)) + if (s >= e || isspace(*s)) break; if ((flag & ARGV_COMMA) && *s == ',') break; @@ -184,7 +187,7 @@ VAV_Parse(const char *s, int *argc, int flag) } if (*s == '"' && !(flag & ARGV_NOESC)) break; - if (*s == '\0') { + if (s >= e) { argv[0] = err_missing_quote; return (argv); } @@ -203,7 +206,7 @@ VAV_Parse(const char *s, int *argc, int flag) } else { argv[nargv++] = VAV_BackSlashDecode(p, s); } - if (*s != '\0') + if (s < e) s++; } argv[nargv] = NULL; @@ -212,6 +215,13 @@ VAV_Parse(const char *s, int *argc, int flag) return (argv); } +char ** +VAV_Parse(const char *s, int *argc, int flag) +{ + + return (VAV_ParseTxt(s, NULL, argc, flag)); +} + void VAV_Free(char **argv) { From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:07 +0000 (UTC) Subject: [6.0] 8700e538c varnishtest: Replace macro_get() with macro_cat() Message-ID: <20240404143307.7BA2510383B@lists.varnish-cache.org> commit 8700e538cfe23c7ea61ca93cb4beea80f4acdf06 Author: Dridi Boukelmoune Date: Thu Jul 1 08:37:15 2021 +0200 varnishtest: Replace macro_get() with macro_cat() The latter operates on a VSB, which is always what call sites are doing anyway. It also takes the responsibility of ignoring unknown macros, in preparation for more responsibilities that will also require the ability to fail a test case. Conflicts: bin/varnishtest/vtc_http.c diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index c970543ec..e9e178ccc 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -187,8 +187,8 @@ macro_undef(struct vtclog *vl, const char *instance, const char *name) AZ(pthread_mutex_unlock(¯o_mtx)); } -char * -macro_get(const char *b, const char *e) +void +macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) { struct macro *m; int l; @@ -205,7 +205,9 @@ macro_get(const char *b, const char *e) retval = malloc(64); AN(retval); VTIM_format(t, retval); - return (retval); + VSB_cat(vsb, retval); + free(retval); + return; } AZ(pthread_mutex_lock(¯o_mtx)); @@ -215,9 +217,19 @@ macro_get(const char *b, const char *e) break; } if (m != NULL) - retval = strdup(m->val); + REPLACE(retval, m->val); AZ(pthread_mutex_unlock(¯o_mtx)); - return (retval); + + if (retval == NULL) { + if (!ign_unknown_macro) + vtc_fatal(vl, "Macro ${%.*s} not found", + (int)(e - b), b); + VSB_printf(vsb, "${%.*s}", (int)(e - b), b); + return; + } + + VSB_cat(vsb, retval); + free(retval); } struct vsb * @@ -242,7 +254,6 @@ macro_expand(struct vtclog *vl, const char *text) { struct vsb *vsb; const char *p, *q; - char *m; vsb = VSB_new_auto(); AN(vsb); @@ -262,19 +273,7 @@ macro_expand(struct vtclog *vl, const char *text) assert(p[1] == '{'); assert(q[0] == '}'); p += 2; - m = macro_get(p, q); - if (m == NULL) { - if (!ign_unknown_macro) { - VSB_destroy(&vsb); - vtc_fatal(vl, "Macro ${%.*s} not found", - (int)(q - p), p); - NEEDLESS(return (NULL)); - } - VSB_printf(vsb, "${%.*s}", (int)(q - p), p); - } else { - VSB_printf(vsb, "%s", m); - free(m); - } + macro_cat(vl, vsb, p, q); text = q + 1; } AZ(VSB_finish(vsb)); @@ -509,7 +508,6 @@ exec_file(const char *fn, const char *script, const char *tmpdir, FILE *f; struct vsb *vsb; const char *p; - char *q; (void)signal(SIGPIPE, SIG_IGN); @@ -529,12 +527,8 @@ exec_file(const char *fn, const char *script, const char *tmpdir, vsb = VSB_new_auto(); AN(vsb); - if (*fn != '/') { - q = macro_get("pwd", NULL); - AN(q); - VSB_cat(vsb, q); - free(q); - } + if (*fn != '/') + macro_cat(vltop, vsb, "pwd", NULL); p = strrchr(fn, '/'); if (p != NULL) { VSB_putc(vsb, '/'); diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index c091afe40..7c705bf29 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -116,7 +116,7 @@ void macro_undef(struct vtclog *vl, const char *instance, const char *name); void macro_def(struct vtclog *vl, const char *instance, const char *name, const char *fmt, ...) v_printflike_(4, 5); -char *macro_get(const char *, const char *); +void macro_cat(struct vtclog *, struct vsb *, const char *, const char *); struct vsb *macro_expand(struct vtclog *vl, const char *text); struct vsb *macro_expandf(struct vtclog *vl, const char *, ...) v_printflike_(2, 3); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 9cd701f8f..828492b6b 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -849,11 +849,9 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, int bodylen = 0; char *b, *c; char *nullbody; - char *m; int nolen = 0; int l; - (void)vl; nullbody = body; for (; *av != NULL; av++) { @@ -926,10 +924,9 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, break; } if (!nohost) { - m = macro_get("localhost", NULL); - AN(m); - VSB_printf(hp->vsb, "Host: %s%s", m, nl); - free(m); + VSB_cat(hp->vsb, "Host: "); + macro_cat(vl, hp->vsb, "localhost", NULL); + VSB_cat(hp->vsb, nl); } if (body != NULL && !nolen) VSB_printf(hp->vsb, "Content-Length: %d%s", bodylen, nl); From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:07 +0000 (UTC) Subject: [6.0] 5f8310330 varnishtest: Allow macros to be backed by functions Message-ID: <20240404143307.97251103846@lists.varnish-cache.org> commit 5f8310330e157400b8b2a817d5b3cb91b05bf442 Author: Dridi Boukelmoune Date: Thu Jun 17 15:49:45 2021 +0200 varnishtest: Allow macros to be backed by functions Instead of having a mere value, these would be able to compute a macro expansion. We parse the contents inside the ${...} delimiters as a VAV, but there can't be (yet?) nested curly {braces}, even quoted. The first argument inside the delimiters is the macro name, and other VAV arguments are treated as arguments to the macro's function. For example ${foo,bar,baz} would call the a macro "foo"'s function with arguments "bar" and "baz". Simple macros don't take arguments and work as usual. Conflicts: bin/varnishtest/vtc.c bin/varnishtest/vtc_main.c diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index e9e178ccc..87ee0055b 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -75,6 +75,7 @@ struct macro { VTAILQ_ENTRY(macro) list; char *name; char *val; + macro_f *func; }; static VTAILQ_HEAD(,macro) macro_list = VTAILQ_HEAD_INITIALIZER(macro_list); @@ -82,10 +83,10 @@ static VTAILQ_HEAD(,macro) macro_list = VTAILQ_HEAD_INITIALIZER(macro_list); /**********************************************************************/ static struct macro * -macro_def_int(const char *name, const char *fmt, va_list ap) +macro_def_int(const char *name, macro_f *func, const char *fmt, va_list ap) { struct macro *m; - char buf[256]; + struct vsb *vsb; VTAILQ_FOREACH(m, ¯o_list, list) if (!strcmp(name, m->name)) @@ -98,9 +99,18 @@ macro_def_int(const char *name, const char *fmt, va_list ap) VTAILQ_INSERT_TAIL(¯o_list, m, list); } AN(m); - vbprintf(buf, fmt, ap); - REPLACE(m->val, buf); - AN(m->val); + if (func != NULL) { + AZ(fmt); + m->func = func; + } else { + vsb = VSB_new_auto(); + AN(vsb); + VSB_vprintf(vsb, fmt, ap); + AZ(VSB_finish(vsb)); + REPLACE(m->val, VSB_data(vsb)); + AN(m->val); + VSB_destroy(&vsb); + } return (m); } @@ -111,12 +121,12 @@ macro_def_int(const char *name, const char *fmt, va_list ap) */ void -extmacro_def(const char *name, const char *fmt, ...) +extmacro_def(const char *name, macro_f *func, const char *fmt, ...) { va_list ap; va_start(ap, fmt); - (void)macro_def_int(name, fmt, ap); + (void)macro_def_int(name, func, fmt, ap); va_end(ap); } @@ -132,8 +142,13 @@ init_macro(void) struct macro *m; /* Dump the extmacros for completeness */ - VTAILQ_FOREACH(m, ¯o_list, list) - vtc_log(vltop, 4, "extmacro def %s=%s", m->name, m->val); + VTAILQ_FOREACH(m, ¯o_list, list) { + if (m->val != NULL) + vtc_log(vltop, 4, + "extmacro def %s=%s", m->name, m->val); + else + vtc_log(vltop, 4, "extmacro def %s(...)", m->name); + } AZ(pthread_mutex_init(¯o_mtx, NULL)); } @@ -155,7 +170,7 @@ macro_def(struct vtclog *vl, const char *instance, const char *name, AZ(pthread_mutex_lock(¯o_mtx)); va_start(ap, fmt); - m = macro_def_int(name, fmt, ap); + m = macro_def_int(name, NULL, fmt, ap); va_end(ap); vtc_log(vl, 4, "macro def %s=%s", name, m->val); AZ(pthread_mutex_unlock(¯o_mtx)); @@ -191,35 +206,61 @@ void macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) { struct macro *m; - int l; - char *retval = NULL; + char **argv, *retval = NULL; + const char *err = NULL; + int argc; AN(b); if (e == NULL) e = strchr(b, '\0'); AN(e); - l = e - b; - if (l == 4 && !memcmp(b, "date", l)) { + argv = VAV_ParseTxt(b, e, &argc, ARGV_COMMA); + AN(argv); + + if (*argv != NULL) + vtc_fatal(vl, "Macro ${%.*s} parsing failed: %s", + (int)(e - b), b, *argv); + + assert(argc >= 2); + + if (!strcmp(argv[1], "date")) { double t = VTIM_real(); retval = malloc(64); AN(retval); VTIM_format(t, retval); VSB_cat(vsb, retval); free(retval); + VAV_Free(argv); return; } AZ(pthread_mutex_lock(¯o_mtx)); VTAILQ_FOREACH(m, ¯o_list, list) { CHECK_OBJ_NOTNULL(m, MACRO_MAGIC); - if (!strncmp(b, m->name, l) && m->name[l] == '\0') + if (!strcmp(argv[1], m->name)) break; } - if (m != NULL) - REPLACE(retval, m->val); + if (m != NULL) { + if (m->func != NULL) { + AZ(m->val); + retval = m->func(argc, argv, &err); + if (err == NULL) + AN(retval); + } else { + AN(m->val); + if (argc == 2) + REPLACE(retval, m->val); + else + err = "macro does not take arguments"; + } + } AZ(pthread_mutex_unlock(¯o_mtx)); + if (err != NULL) + vtc_fatal(vl, "Macro ${%.*s} failed: %s", + (int)(e - b), b, err); + if (retval == NULL) { if (!ign_unknown_macro) vtc_fatal(vl, "Macro ${%.*s} not found", @@ -230,6 +271,7 @@ macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) VSB_cat(vsb, retval); free(retval); + VAV_Free(argv); } struct vsb * diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 7c705bf29..868fe6884 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -121,8 +121,9 @@ struct vsb *macro_expand(struct vtclog *vl, const char *text); struct vsb *macro_expandf(struct vtclog *vl, const char *, ...) v_printflike_(2, 3); -void extmacro_def(const char *name, const char *fmt, ...) - v_printflike_(2, 3); +typedef char* macro_f(int, char *const *, const char **); +void extmacro_def(const char *name, macro_f *func, const char *fmt, ...) + v_printflike_(3, 4); struct http; void cmd_stream(CMD_ARGS); diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 27754d04e..ca263aa61 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -161,7 +161,7 @@ parse_D_opt(char *arg) if (!q) return (0); *q++ = '\0'; - extmacro_def(p, "%s", q); + extmacro_def(p, NULL, "%s", q); return (1); } @@ -504,7 +504,8 @@ i_mode(void) } AN(topbuild); - extmacro_def("topbuild", "%s", topbuild); + extmacro_def("topbuild", NULL, "%s", topbuild); + /* * Build $PATH which can find all programs in the build tree */ @@ -571,7 +572,7 @@ ip_magic(void) fd = VTCP_bind(sa, NULL); assert(fd >= 0); VTCP_myname(fd, abuf, sizeof abuf, pbuf, sizeof(pbuf)); - extmacro_def("localhost", "%s", abuf); + extmacro_def("localhost", NULL, "%s", abuf); #if defined (__APPLE__) /* @@ -583,7 +584,7 @@ ip_magic(void) #endif /* Expose a backend that is forever down. */ - extmacro_def("bad_backend", "%s %s", abuf, pbuf); + extmacro_def("bad_backend", NULL, "%s %s", abuf, pbuf); /* * We need an IP number which will not repond, ever, and that is a @@ -593,7 +594,7 @@ ip_magic(void) * check your /proc/sys/net/ipv4/ip_nonlocal_bind setting. */ - extmacro_def("bad_ip", "%s", "192.0.2.255"); + extmacro_def("bad_ip", NULL, "%s", "192.0.2.255"); } /********************************************************************** @@ -673,7 +674,7 @@ main(int argc, char * const *argv) tmppath = strdup("/tmp"); cwd = getcwd(buf, sizeof buf); - extmacro_def("pwd", "%s", cwd); + extmacro_def("pwd", NULL, "%s", cwd); vmod_path = NULL; From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:07 +0000 (UTC) Subject: [6.0] 2f6d9be42 varnishtest: Turn the ${date} macro into a function Message-ID: <20240404143307.B66C5103855@lists.varnish-cache.org> commit 2f6d9be42ee56ffc6654c0b02baa9238572b09e0 Author: Dridi Boukelmoune Date: Thu Jun 17 16:10:16 2021 +0200 varnishtest: Turn the ${date} macro into a function Conflicts: bin/varnishtest/vtc.c diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 87ee0055b..4b1d415ff 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -41,7 +41,6 @@ #include "vav.h" #include "vrnd.h" -#include "vtim.h" #define MAX_TOKENS 200 @@ -224,17 +223,6 @@ macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) assert(argc >= 2); - if (!strcmp(argv[1], "date")) { - double t = VTIM_real(); - retval = malloc(64); - AN(retval); - VTIM_format(t, retval); - VSB_cat(vsb, retval); - free(retval); - VAV_Free(argv); - return; - } - AZ(pthread_mutex_lock(¯o_mtx)); VTAILQ_FOREACH(m, ¯o_list, list) { CHECK_OBJ_NOTNULL(m, MACRO_MAGIC); diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index ca263aa61..e2ccbef24 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -597,6 +597,32 @@ ip_magic(void) extmacro_def("bad_ip", NULL, "%s", "192.0.2.255"); } +/********************************************************************** + * Macros + */ + +static char * v_matchproto_(macro_f) +macro_func_date(int argc, char *const *argv, const char **err) +{ + double t; + char *s; + + assert(argc >= 2); + AN(argv); + AN(err); + + if (argc > 2) { + *err = "macro does not take arguments"; + return (NULL); + } + + t = VTIM_real(); + s = malloc(VTIM_FORMAT_SIZE); + AN(s); + VTIM_format(t, s); + return (s); +} + /********************************************************************** * Main */ @@ -676,6 +702,8 @@ main(int argc, char * const *argv) cwd = getcwd(buf, sizeof buf); extmacro_def("pwd", NULL, "%s", cwd); + extmacro_def("date", macro_func_date, NULL); + vmod_path = NULL; params_vsb = VSB_new_auto(); From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:07 +0000 (UTC) Subject: [6.0] b9f2b4733 varnishtest: New ${string, [, ...]} macro Message-ID: <20240404143307.E1EC510385A@lists.varnish-cache.org> commit b9f2b473376754b64eea0d501cccaeb3ef75c54c Author: Dridi Boukelmoune Date: Thu Jun 17 17:08:11 2021 +0200 varnishtest: New ${string,[,...]} macro Its first action ${string,repeat,,} helps simplify many unwieldy test cases that will hopefully be easier to edit from now on. Conflicts: bin/varnishtest/tests/c00071.vtc bin/varnishtest/tests/r02219.vtc bin/varnishtest/vtc_main.c diff --git a/bin/varnishtest/tests/c00027.vtc b/bin/varnishtest/tests/c00027.vtc index 5c22daf1d..246aef15f 100644 --- a/bin/varnishtest/tests/c00027.vtc +++ b/bin/varnishtest/tests/c00027.vtc @@ -3,19 +3,19 @@ varnishtest "Test eviction" server s1 { rxreq expect req.url == "/1" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" rxreq expect req.url == "/2" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" rxreq expect req.url == "/3" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" rxreq expect req.url == "/4" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" rxreq expect req.url == "/5" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" } -start varnish v1 -arg "-s default,1M" -vcl+backend { diff --git a/bin/varnishtest/tests/c00071.vtc b/bin/varnishtest/tests/c00071.vtc index 6cff35d87..4d9ccd603 100644 --- a/bin/varnishtest/tests/c00071.vtc +++ b/bin/varnishtest/tests/c00071.vtc @@ -21,7 +21,7 @@ varnish v1 -vcl+backend { vtc.workspace_alloc(client, -10); } else if (req.url ~ "/baz") { - set resp.http.x-foo = regsub(req.url, "baz", "baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaz"); + set resp.http.x-foo = regsub(req.url, "baz", "b${string,repeat,77,a}z"); } set resp.http.x-of = vtc.workspace_overflowed(client); } diff --git a/bin/varnishtest/tests/o00000.vtc b/bin/varnishtest/tests/o00000.vtc index 08459fbe4..c6691ba3c 100644 --- a/bin/varnishtest/tests/o00000.vtc +++ b/bin/varnishtest/tests/o00000.vtc @@ -199,7 +199,7 @@ varnish v1 -vsl_catchup # Try with appended request (See also: #1728) client c2 { - send "PROXY TCP6 1:f::3 5:a::8 1234 5678\r\nGET /3 HTTP/1.1\r\nHost: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n\r\n" + send "PROXY TCP6 1:f::3 5:a::8 1234 5678\r\nGET /3 HTTP/1.1\r\nHost: ${string,repeat,54,a}\r\n\r\n" rxresp expect resp.http.url == "/3" } -run @@ -217,7 +217,7 @@ varnish v1 -vsl_catchup # Malformed, too long (106) # NB: Should check VSL for proper disposal client c2 { - send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 \r\n" + send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 ${string,repeat,68," "}\r\n" expect_close } -run @@ -226,7 +226,7 @@ varnish v1 -vsl_catchup # Malformed, too long (107) # NB: Should check VSL for proper disposal client c2 { - send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 \r\n" + send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 ${string,repeat,69," "}\r\n" expect_close } -run @@ -235,7 +235,7 @@ varnish v1 -vsl_catchup # Malformed, too long (108) # NB: Should check VSL for proper disposal client c2 { - send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 \r\n" + send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 ${string,repeat,70," "}\r\n" expect_close } -run diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc index b09a8fcaf..011e9931c 100644 --- a/bin/varnishtest/tests/o00005.vtc +++ b/bin/varnishtest/tests/o00005.vtc @@ -215,25 +215,7 @@ client c1 { 01 bb 03 00 04 c5 1b 5b 2b 01 00 02 68 32 - 02 00 c8 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 + 02 00 c8 ${string,repeat,200,"61 "} 20 00 3d 01 00 00 00 00 21 00 07 54 4c 53 76 31 2e 33 diff --git a/bin/varnishtest/tests/r00498.vtc b/bin/varnishtest/tests/r00498.vtc index c9f2520b5..c11df3f77 100644 --- a/bin/varnishtest/tests/r00498.vtc +++ b/bin/varnishtest/tests/r00498.vtc @@ -3,7 +3,7 @@ varnishtest "very very very long return header" server s1 { rxreq expect req.url == "/" - txresp -hdr "Location: 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" -body {foo} + txresp -hdr "Location: ${string,repeat,8136,1}" -body {foo} } -start varnish v1 -vcl+backend { diff --git a/bin/varnishtest/tests/r01120.vtc b/bin/varnishtest/tests/r01120.vtc index c55064a00..d906096d7 100644 --- a/bin/varnishtest/tests/r01120.vtc +++ b/bin/varnishtest/tests/r01120.vtc @@ -18,7 +18,7 @@ client c1 { rxresp expect resp.bodylen == 4 #txreq -hdr "Foo: blablaA" - txreq -hdr "Foo: blablaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + txreq -hdr "Foo: blabla${string,repeat,1430,A}" rxresp expect resp.bodylen == 5 } -run diff --git a/bin/varnishtest/tests/r01274.vtc b/bin/varnishtest/tests/r01274.vtc index 90cd95deb..d90e98273 100644 --- a/bin/varnishtest/tests/r01274.vtc +++ b/bin/varnishtest/tests/r01274.vtc @@ -3,10 +3,10 @@ varnishtest "#1274 - panic when Vary field-name is too large to fit in a signed server s1 { rxreq # Vary header more than 127 characters long - txresp -hdr "Vary: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -bodylen 9 + txresp -hdr "Vary: ${string,repeat,200,a}" -bodylen 9 rxreq # Vary header more than 127 characters long - txresp -hdr "Vary: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -bodylen 8 + txresp -hdr "Vary: ${string,repeat,200,a}" -bodylen 8 } -start varnish v1 -vcl+backend { } -start diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc index d899fac54..e188e69a5 100644 --- a/bin/varnishtest/tests/r01284.vtc +++ b/bin/varnishtest/tests/r01284.vtc @@ -6,7 +6,7 @@ server s1 { txresp -bodylen 1048290 rxreq expect req.url == "/obj2" - txresp -hdr "Long: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -hdr "Long2: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + txresp -hdr "Long: ${string,repeat,127,a}" -hdr "Long2: ${string,repeat,135,a}" } -start varnish v1 \ diff --git a/bin/varnishtest/tests/r02219.vtc b/bin/varnishtest/tests/r02219.vtc index 8b4d1c188..8ee8801ff 100644 --- a/bin/varnishtest/tests/r02219.vtc +++ b/bin/varnishtest/tests/r02219.vtc @@ -19,7 +19,7 @@ varnish v1 -arg "-p workspace_client=9k" -proto PROXY -vcl+backend { } -start client c1 { - send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA HTTP/1.1\r\n\r\n" + send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /${string,repeat,771,A} HTTP/1.1\r\n\r\n" rxresp } -run @@ -29,43 +29,13 @@ client c2 { 0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a 21 00 00 00 47 45 54 20 2f -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 +${string,repeat,764,"42 "} 20 48 54 54 50 2f 31 2e 31 0d 0a 0d 0a } rxresp } -run client c3 { - send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC HTTP/1.1\r\n\r\n" + send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /${string,repeat,796,C} HTTP/1.1\r\n\r\n" rxresp } -run diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index e2ccbef24..b59f59778 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -623,6 +623,59 @@ macro_func_date(int argc, char *const *argv, const char **err) return (s); } +static char * +macro_func_string_repeat(int argc, char *const *argv, const char **err) +{ + struct vsb vsb[1]; + char *p, *res; + size_t l; + int i; + + if (argc != 4) { + *err = "repeat takes 2 arguments"; + return (NULL); + } + + i = strtod(argv[2], &p); + + if (p == argv[2] || *p != '\0' || i < 0) { + *err = "invalid number of repetitions"; + return (NULL); + } + + l = (strlen(argv[3]) * i) + 1; + res = malloc(l); + AN(res); + AN(VSB_new(vsb, res, l, VSB_FIXEDLEN)); + while (i > 0) { + AZ(VSB_cat(vsb, argv[3])); + i--; + } + AZ(VSB_finish(vsb)); + VSB_delete(vsb); + return (res); +} + +static char * +macro_func_string(int argc, char *const *argv, const char **err) +{ + + assert(argc >= 2); + AN(argv); + AN(err); + + if (argc == 2) { + *err = "missing action"; + return (NULL); + } + + if (!strcmp(argv[2], "repeat")) + return (macro_func_string_repeat(argc - 1, argv + 1, err)); + + *err = "unknown action"; + return (NULL); +} + /********************************************************************** * Main */ @@ -703,6 +756,7 @@ main(int argc, char * const *argv) extmacro_def("pwd", NULL, "%s", cwd); extmacro_def("date", macro_func_date, NULL); + extmacro_def("string", macro_func_string, NULL); vmod_path = NULL; From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:08 +0000 (UTC) Subject: [6.0] 21ec091a0 doc: Add varnishtest macro syntax and built-in macros Message-ID: <20240404143308.0BC2710385E@lists.varnish-cache.org> commit 21ec091a03b18a8ecb68ed330f551237e7bd00ac Author: Dridi Boukelmoune Date: Thu Jul 1 12:59:33 2021 +0200 doc: Add varnishtest macro syntax and built-in macros diff --git a/doc/sphinx/reference/vtc.rst b/doc/sphinx/reference/vtc.rst index 876b80900..30d53cf82 100644 --- a/doc/sphinx/reference/vtc.rst +++ b/doc/sphinx/reference/vtc.rst @@ -52,6 +52,64 @@ being the command and the following ones being its arguments. To continue over to a new line without breaking the argument string, you can escape the newline character (\\n) with a backslash (\\). +MACROS +====== + +When a string is processed, macro expansion is performed. Macros are in the +form ``${[,...]}``, they have a name followed by an optional +comma- or space-separated list of arguments. Leading and trailing spaces are +ignored. + +The macros ``${foo,bar,baz}`` and ``${ foo bar baz }`` are equivalent. If an +argument contains a space or a comma, arguments can be quoted. For example the +macro ``${foo,"bar,baz"}`` gives one argument ``bar,baz`` to the macro called +``foo``. + +Unless documented otherwise, all macros are simple macros that don't take +arguments. + +Built-in macros +--------------- + +``${bad_backend}`` + A socket address that will reliably never accept connections. + +``${bad_ip}`` + An unlikely IPv4 address. + +``${date}`` + The current date and time formatted for HTTP. + +``${listen_addr}`` + The default listen address various components use, by default a random + port on localhost. + +``${localhost}`` + The first IP address that resolves to "localhost". + +``${pwd}`` + The working directory from which ``varnishtest`` was executed. + +``${string,[,...]}`` + The ``string`` macro is the entry point for text generation, it takes + a specialized action with each its own set of arguments. + +``${string,repeat,,}`` + Repeat ``uint`` times the string ``str``. + +``${testdir}`` + The directory containing the VTC script of the ongoing test case + execution. + +``${tmpdir}`` + The dedicated working directory for the ongoing test case execution, + which happens to also be the current working directory. Useful when an + absolute path to the working directory is needed. + +``${topbuild}`` + Only present when the ``-i`` option is used, to work on Varnish itself + instead of a regular installation. + SYNTAX ====== From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:08 +0000 (UTC) Subject: [6.0] f7d8112ca varnishtest: New macro_isdef() function Message-ID: <20240404143308.29730103863@lists.varnish-cache.org> commit f7d8112ca77f6d424faa1fc6dbf4087d581cdb1c Author: Dridi Boukelmoune Date: Fri Apr 14 15:22:30 2023 +0200 varnishtest: New macro_isdef() function diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 4b1d415ff..a1e1c9c44 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -201,6 +201,26 @@ macro_undef(struct vtclog *vl, const char *instance, const char *name) AZ(pthread_mutex_unlock(¯o_mtx)); } +unsigned +macro_isdef(const char *instance, const char *name) +{ + char buf1[256]; + struct macro *m; + + if (instance != NULL) { + bprintf(buf1, "%s_%s", instance, name); + name = buf1; + } + + AZ(pthread_mutex_lock(¯o_mtx)); + VTAILQ_FOREACH(m, ¯o_list, list) + if (!strcmp(name, m->name)) + break; + AZ(pthread_mutex_unlock(¯o_mtx)); + + return (m != NULL); +} + void macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) { diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 868fe6884..80319d8a9 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -114,8 +114,8 @@ int exec_file(const char *fn, const char *script, const char *tmpdir, void macro_undef(struct vtclog *vl, const char *instance, const char *name); void macro_def(struct vtclog *vl, const char *instance, const char *name, - const char *fmt, ...) - v_printflike_(4, 5); + const char *fmt, ...) v_printflike_(4, 5); +unsigned macro_isdef(const char *instance, const char *name); void macro_cat(struct vtclog *, struct vsb *, const char *, const char *); struct vsb *macro_expand(struct vtclog *vl, const char *text); struct vsb *macro_expandf(struct vtclog *vl, const char *, ...) From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:08 +0000 (UTC) Subject: [6.0] 61f6a3774 cache: Move the pdiff() function to vas.h Message-ID: <20240404143308.442BC10386F@lists.varnish-cache.org> commit 61f6a377487b6f830f863c34c0cc7d930e1293d1 Author: Dridi Boukelmoune Date: Wed Nov 25 17:22:42 2020 +0100 cache: Move the pdiff() function to vas.h This way it can be used almost anywhere in the code base and out of tree, not just in the cache process. The function was slightly polished after prior discussions with phk and slink. Conflicts: include/vas.h diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b26a2e73a..f5329aafd 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -799,20 +799,6 @@ void RFC2616_Weaken_Etag(struct http *hp); void RFC2616_Vary_AE(struct http *hp); void RFC2616_Response_Body(const struct worker *, const struct busyobj *); -/* - * A normal pointer difference is signed, but we never want a negative value - * so this little tool will make sure we don't get that. - */ - -static inline unsigned -pdiff(const void *b, const void *e) -{ - - assert(b <= e); - return - ((unsigned)((const unsigned char *)e - (const unsigned char *)b)); -} - #define Tcheck(t) do { \ AN((t).b); \ AN((t).e); \ diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 5de7a6830..a691860c4 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -39,7 +39,7 @@ WS_Assert(const struct ws *ws) { CHECK_OBJ_NOTNULL(ws, WS_MAGIC); - DSL(DBG_WORKSPACE, 0, "WS(%p) = (%s, %p %u %u %u)", + DSL(DBG_WORKSPACE, 0, "WS(%p) = (%s, %p %zu %zu %zu)", ws, ws->id, ws->s, pdiff(ws->s, ws->f), ws->r == NULL ? 0 : pdiff(ws->f, ws->r), pdiff(ws->s, ws->e)); @@ -268,7 +268,7 @@ WS_ReserveSize(struct ws *ws, unsigned bytes) return (0); } ws->r = ws->f + b2; - DSL(DBG_WORKSPACE, 0, "WS_ReserveSize(%p, %u/%u) = %u", + DSL(DBG_WORKSPACE, 0, "WS_ReserveSize(%p, %u/%u) = %zu", ws, b2, bytes, pdiff(ws->f, ws->r)); WS_Assert(ws); return (pdiff(ws->f, ws->r)); @@ -294,7 +294,7 @@ WS_Reserve(struct ws *ws, unsigned bytes) return (0); } ws->r = ws->f + b2; - DSL(DBG_WORKSPACE, 0, "WS_Reserve(%p, %u/%u) = %u", + DSL(DBG_WORKSPACE, 0, "WS_Reserve(%p, %u/%u) = %zu", ws, b2, bytes, pdiff(ws->f, ws->r)); WS_Assert(ws); return (pdiff(ws->f, ws->r)); diff --git a/include/vas.h b/include/vas.h index 11c00692d..ccbbbec32 100644 --- a/include/vas.h +++ b/include/vas.h @@ -88,4 +88,17 @@ do { \ "", VAS_INCOMPLETE); \ } while (0) +/* + * A normal pointer difference is signed, but when we don't want a negative + * value this little tool will make sure we don't get that. + */ + +static inline size_t +pdiff(const void *b, const void *e) +{ + + assert(b <= e); + return ((size_t)((const char *)e - (const char *)b)); +} + #endif From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:08 +0000 (UTC) Subject: [6.0] 382f84e2b cache: Move the txt declaration to vdef.h Message-ID: <20240404143308.604C0103877@lists.varnish-cache.org> commit 382f84e2bc0635fb3097a83fe3570a3425c719e0 Author: Dridi Boukelmoune Date: Tue Aug 3 10:06:48 2021 +0200 cache: Move the txt declaration to vdef.h There are other components than the cache process that could benefit from it, in particular libvcc. The relationship with vas.h is somewhat unfortunate but that also centralize the Tcheck() logic in pdiff() and doesn't actually require vas.h unless the macros are used, in which case it's almost guaranteed that the calling code already included vas.h in the first place. The benefits should outweigh the drawbacks. Conflicts: bin/varnishd/cache/cache.h bin/varnishtest/vtc_http.c diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f5329aafd..21682ec3b 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -105,13 +105,6 @@ struct listen_sock; /*--------------------------------------------------------------------*/ -typedef struct { - const char *b; - const char *e; -} txt; - -/*--------------------------------------------------------------------*/ - enum req_step { R_STP_NONE = 0, #define REQ_STEP(l, u, arg) R_STP_##u, @@ -799,24 +792,6 @@ void RFC2616_Weaken_Etag(struct http *hp); void RFC2616_Vary_AE(struct http *hp); void RFC2616_Response_Body(const struct worker *, const struct busyobj *); -#define Tcheck(t) do { \ - AN((t).b); \ - AN((t).e); \ - assert((t).b <= (t).e); \ - } while(0) - -/* - * unsigned length of a txt - */ - -static inline unsigned -Tlen(const txt t) -{ - - Tcheck(t); - return ((unsigned)(t.e - t.b)); -} - /* * We want to cache the most recent timestamp in wrk->lastused to avoid * extra timestamps in cache_pool.c. Hide this detail with a macro diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 828492b6b..99c5f2770 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -795,7 +795,7 @@ cmd_http_gunzip(CMD_ARGS) */ static void -gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen) +gzip_body(const struct http *hp, const char *text, char **body, int *bodylen) { int l; z_stream vz; @@ -805,11 +805,11 @@ gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen) memset(&vz, 0, sizeof vz); - l = strlen(txt); + l = strlen(text); *body = calloc(1, l + OVERHEAD); AN(*body); - vz.next_in = TRUST_ME(txt); + vz.next_in = TRUST_ME(text); vz.avail_in = l; vz.next_out = TRUST_ME(*body); diff --git a/include/vas.h b/include/vas.h index ccbbbec32..4633d64a2 100644 --- a/include/vas.h +++ b/include/vas.h @@ -97,6 +97,8 @@ static inline size_t pdiff(const void *b, const void *e) { + AN(b); + AN(e); assert(b <= e); return ((size_t)((const char *)e - (const char *)b)); } diff --git a/include/vdef.h b/include/vdef.h index 128f44148..20ce9a63f 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -229,3 +229,15 @@ typedef double vtim_mono; typedef double vtim_real; typedef double vtim_dur; + +/********************************************************************** + * txt (vas.h needed for the macros) + */ + +typedef struct { + const char *b; + const char *e; +} txt; + +#define Tcheck(t) do { (void)pdiff((t).b, (t).e); } while (0) +#define Tlen(t) (pdiff((t).b, (t).e)) From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:08 +0000 (UTC) Subject: [6.0] 6fb49fee9 Flexelint a signed/unsigned mix. Message-ID: <20240404143308.7C52610388B@lists.varnish-cache.org> commit 6fb49fee9d58a32a1a88e6993d10d61a31703642 Author: Poul-Henning Kamp Date: Mon Feb 27 13:32:17 2023 +0000 Flexelint a signed/unsigned mix. diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 45cddb34b..e5fcac7af 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -97,7 +97,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) int disallow_empty; unsigned n; char *p; - int i; + unsigned u; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); AN(b); @@ -123,7 +123,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) disallow_empty = 1; /* First field cannot contain SP or CTL */ - for (p = b, i = 0; i < len; p++, i++) { + for (p = b, u = 0; u < len; p++, u++) { if (vct_issp(*p) || vct_isctl(*p)) return (H2SE_PROTOCOL_ERROR); } @@ -134,7 +134,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) disallow_empty = 1; /* Second field cannot contain LWS or CTL */ - for (p = b, i = 0; i < len; p++, i++) { + for (p = b, u = 0; u < len; p++, u++) { if (vct_islws(*p) || vct_isctl(*p)) return (H2SE_PROTOCOL_ERROR); } @@ -155,13 +155,13 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) n = hp->nhd; *flags |= H2H_DECODE_FLAG_SCHEME_SEEN; - for (p = b + namelen, i = 0; i < len-namelen; - p++, i++) { + for (p = b + namelen, u = 0; u < len-namelen; + p++, u++) { if (vct_issp(*p) || vct_isctl(*p)) return (H2SE_PROTOCOL_ERROR); } - if (!i) + if (!u) return (H2SE_PROTOCOL_ERROR); } else if (!strncmp(b, ":authority: ", namelen)) { b+=6; From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:08 +0000 (UTC) Subject: [6.0] 7e51625d3 h2: Manage missing :scheme as a custom error Message-ID: <20240404143308.985C8103895@lists.varnish-cache.org> commit 7e51625d384f4afdf5f7e258b6881fe5c47525e3 Author: Dridi Boukelmoune Date: Mon Dec 4 18:30:06 2023 +0100 h2: Manage missing :scheme as a custom error There is room for further improvement in the dynamic between HPACK and the HTTP/2 session, but this will serve as the first step. Conflicts: include/tbl/h2_error.h diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 201875409..53339dbe5 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -221,14 +221,12 @@ vtr_deliver_f h2_deliver; vtr_minimal_response_f h2_minimal_response; #endif /* TRANSPORT_MAGIC */ -#define H2H_DECODE_FLAG_SCHEME_SEEN 0x1 - /* http2/cache_http2_hpack.c */ struct h2h_decode { unsigned magic; #define H2H_DECODE_MAGIC 0xd092bde4 - int flags; + unsigned has_scheme:1; h2_error error; enum vhd_ret_e vhd_ret; char *out; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index e5fcac7af..e12562523 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -90,7 +90,8 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } static h2_error -h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) +h2h_addhdr(struct h2h_decode *d, struct http *hp, char *b, size_t namelen, + size_t len) { /* XXX: This might belong in cache/cache_http.c */ const char *b0; @@ -142,7 +143,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ - if (*flags & H2H_DECODE_FLAG_SCHEME_SEEN) { + if (d->has_scheme) { VSLb(hp->vsl, SLT_BogoHeader, "Duplicate pseudo-header %.*s%.*s", (int)namelen, b0, @@ -153,7 +154,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) b++; len-=1; n = hp->nhd; - *flags |= H2H_DECODE_FLAG_SCHEME_SEEN; + d->has_scheme = 1; for (p = b + namelen, u = 0; u < len-namelen; p++, u++) { @@ -256,6 +257,9 @@ h2h_decode_fini(const struct h2_sess *h2) VSLb(h2->new_req->http->vsl, SLT_BogoHeader, "HPACK compression error/fini (%s)", VHD_Error(d->vhd_ret)); ret = H2CE_COMPRESSION_ERROR; + } else if (d->error == NULL && !d->has_scheme) { + VSLb(h2->vsl, SLT_Debug, "Missing :scheme"); + ret = H2SE_MISSING_SCHEME; //rfc7540,l,3087,3090 } else ret = d->error; d->magic = 0; @@ -339,8 +343,8 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out_u); if (d->error) break; - d->error = h2h_addhdr(hp, d->out, d->namelen, d->out_u, - &d->flags); + d->error = h2h_addhdr(d, hp, d->out, + d->namelen, d->out_u); if (d->error) break; d->out[d->out_u++] = '\0'; /* Zero guard */ diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index cc0335144..4d2d7134b 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -619,13 +619,11 @@ static h2_error h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { - int scheme_seen; h2_error h2e; ssize_t cl; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); - scheme_seen = h2->decode->flags & H2H_DECODE_FLAG_SCHEME_SEEN; h2e = h2h_decode_fini(h2); h2->new_req = NULL; if (r2->req->req_body_status == REQ_BODY_NONE) { @@ -687,11 +685,6 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 } - if (!(scheme_seen)) { - VSLb(h2->vsl, SLT_Debug, "Missing :scheme"); - return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 - } - AN(req->http->hd[HTTP_HDR_PROTO].b); req->req_step = R_STP_TRANSPORT; diff --git a/include/tbl/h2_error.h b/include/tbl/h2_error.h index 6c14a909b..49436f861 100644 --- a/include/tbl/h2_error.h +++ b/include/tbl/h2_error.h @@ -169,6 +169,15 @@ H2_ERROR( /* descr */ "http/2 rapid reset detected" ) +H2_ERROR( + /* name */ MISSING_SCHEME, + /* val */ 1, /* PROTOCOL_ERROR */ + /* types */ 2, + /* goaway */ 1, + /* reason */ SC_NULL, + /* descr */ "Missing :scheme pseudo-header" +) + H2_ERROR( /* name */ BROKE_WINDOW, /* val */ 8, /* CANCEL */ From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:08 +0000 (UTC) Subject: [6.0] 1c086fbab vtc_http2: Print headers as "name: value" Message-ID: <20240404143308.BA7D910389F@lists.varnish-cache.org> commit 1c086fbabf7167fa976d2d4bba9b63de0815010a Author: Dridi Boukelmoune Date: Wed Mar 27 11:54:43 2024 +0100 vtc_http2: Print headers as "name: value" The extra space before the colon looked uncanny. The rest is just code indentation improvements. Better diff with the --ignore-all-space --word-diff options. diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 491a1d072..7c6a4e563 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -436,22 +436,20 @@ decode_hdr(struct http *hp, struct hpk_hdr *h, const struct vsb *vsb) r = HPK_DecHdr(iter, h + n); if (r == hpk_err ) break; - vtc_log(hp->vl, 4, - "header[%2d]: %s : %s", - n, - h[n].key.ptr, - h[n].value.ptr); + vtc_log(hp->vl, 4, "header[%2d]: %s: %s", + n, h[n].key.ptr, h[n].value.ptr); n++; if (r == hpk_done) break; } - if (r != hpk_done) + if (r != hpk_done) { vtc_log(hp->vl, hp->fatal ? 4 : 0, - "Header decoding failed (%d) %d", r, hp->fatal); - else if (n == MAX_HDR) + "Header decoding failed (%d) %d", r, hp->fatal); + } else if (n == MAX_HDR) { vtc_log(hp->vl, hp->fatal, - "Max number of headers reached (%d)", MAX_HDR); + "Max number of headers reached (%d)", MAX_HDR); + } HPK_FreeIter(iter); } From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:08 +0000 (UTC) Subject: [6.0] d491c3349 txt: New macros to work with strings Message-ID: <20240404143308.D3DC61038AC@lists.varnish-cache.org> commit d491c3349ebf9f900f74a2a1d95b505555191929 Author: Dridi Boukelmoune Date: Wed Mar 27 15:19:01 2024 +0100 txt: New macros to work with strings Conflicts: include/vdef.h diff --git a/include/vdef.h b/include/vdef.h index 20ce9a63f..86fdb0a21 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -241,3 +241,5 @@ typedef struct { #define Tcheck(t) do { (void)pdiff((t).b, (t).e); } while (0) #define Tlen(t) (pdiff((t).b, (t).e)) +#define Tstr(s) ((txt){(s), (s) + strlen(s)}) +#define Tstrcmp(t, s) (strncmp((t).b, (s), Tlen(t))) From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:08 +0000 (UTC) Subject: [6.0] e68d3209b Use the right version of the vct_ishdrval() macro Message-ID: <20240404143308.F172B1038BD@lists.varnish-cache.org> commit e68d3209bc56ea68854b8fdf6afdc322fdaac6e5 Author: Nils Goroll Date: Fri Oct 9 14:36:42 2020 +0200 Use the right version of the vct_ishdrval() macro I accidentally committed an earlier, wrong version. o/ Dridi Ref #3407 https://github.com/varnishcache/varnish-cache/pull/3407#issuecomment-696146625 Conflicts: include/vct.h The previous commit introduces more than just this macro and was skipped on purpose. diff --git a/include/vct.h b/include/vct.h index 1b7ffbd4f..a6edd4bd2 100644 --- a/include/vct.h +++ b/include/vct.h @@ -75,6 +75,8 @@ vct_is(int x, uint16_t y) #define vct_isxmlnamestart(x) vct_is(x, VCT_XMLNAMESTART) #define vct_isxmlname(x) vct_is(x, VCT_XMLNAMESTART | VCT_XMLNAME) #define vct_istchar(x) vct_is(x, VCT_ALPHA | VCT_DIGIT | VCT_TCHAR) +#define vct_ishdrval(x) \ + (((uint8_t)(x) >= 0x20 && (uint8_t)(x) != 0x7f) ||(uint8_t)(x) == 0x09) static inline int vct_iscrlf(const char* p, const char* end) From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:09 +0000 (UTC) Subject: [6.0] e9edf57bb h2: Improve pseudo-header handling Message-ID: <20240404143309.19BCC1038D2@lists.varnish-cache.org> commit e9edf57bb0349dfbfa6d311ebb0e8cab15cc2a7d Author: Dag Haavi Finstad Date: Fri Mar 10 19:59:43 2023 +0100 h2: Improve pseudo-header handling diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index e12562523..055cb3e33 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -134,6 +134,15 @@ h2h_addhdr(struct h2h_decode *d, struct http *hp, char *b, size_t namelen, n = HTTP_HDR_URL; disallow_empty = 1; + // rfc7540,l,3060,3071 + if ((len > 0 && *b != '/') || + (len > 1 && *(b+1) == '/')) { + VSLb(hp->vsl, SLT_BogoHeader, + "Illegal :path pseudo-header %.*s", + (int)len, b); + return (H2SE_PROTOCOL_ERROR); + } + /* Second field cannot contain LWS or CTL */ for (p = b, u = 0; u < len; p++, u++) { if (vct_islws(*p) || vct_isctl(*p)) diff --git a/bin/varnishtest/tests/a02027.vtc b/bin/varnishtest/tests/a02027.vtc new file mode 100644 index 000000000..731f72aca --- /dev/null +++ b/bin/varnishtest/tests/a02027.vtc @@ -0,0 +1,29 @@ +varnishtest "Malformed :path handling" + +server s1 { +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return (synth(200)); + } +} -start +varnish v1 -cliok "param.set feature +http2" + +client c1 { + stream 1 { + txreq -noadd -hdr ":authority" "foo.com" -hdr ":path" "foobar" -hdr ":scheme" "http" -hdr ":method" "GET" + rxrst + expect rst.err == PROTOCOL_ERROR + } -run + +} -run + +client c1 { + stream 1 { + txreq -noadd -hdr ":authority" "foo.com" -hdr ":path" "//foo" -hdr ":scheme" "http" -hdr ":method" "GET" + rxrst + expect rst.err == PROTOCOL_ERROR + } -run + +} -run From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:09 +0000 (UTC) Subject: [6.0] 3643aab3b h2: Allow :path * for OPTIONS Message-ID: <20240404143309.353331038DC@lists.varnish-cache.org> commit 3643aab3bef97006247dd2fd2d5d0bbb0a9abf5f Author: Dag Haavi Finstad Date: Fri Mar 10 19:59:50 2023 +0100 h2: Allow :path * for OPTIONS Conflicts: bin/varnishd/http2/cache_http2_proto.c diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 055cb3e33..1e46c4b8d 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -135,8 +135,9 @@ h2h_addhdr(struct h2h_decode *d, struct http *hp, char *b, size_t namelen, disallow_empty = 1; // rfc7540,l,3060,3071 - if ((len > 0 && *b != '/') || - (len > 1 && *(b+1) == '/')) { + if (((len > 0 && *b != '/') || + (len > 1 && *(b+1) == '/')) && + (strncmp(b, "*", len) != 0)) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal :path pseudo-header %.*s", (int)len, b); diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 4d2d7134b..f7c4c5973 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -687,6 +687,13 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, AN(req->http->hd[HTTP_HDR_PROTO].b); + if (*req->http->hd[HTTP_HDR_URL].b == '*' && + (Tlen(req->http->hd[HTTP_HDR_METHOD]) != 7 || + strncmp(req->http->hd[HTTP_HDR_METHOD].b, "OPTIONS", 7))) { + VSLb(h2->vsl, SLT_BogoHeader, "Illegal :path pseudo-header"); + return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3068,3071 + } + req->req_step = R_STP_TRANSPORT; req->task.func = h2_do_req; req->task.priv = req; diff --git a/bin/varnishtest/tests/a02027.vtc b/bin/varnishtest/tests/a02027.vtc index 731f72aca..ff34b0071 100644 --- a/bin/varnishtest/tests/a02027.vtc +++ b/bin/varnishtest/tests/a02027.vtc @@ -27,3 +27,43 @@ client c1 { } -run } -run + +client c1 { + stream 3 { + txreq -noadd -hdr ":authority" "foo.com" -hdr ":path" "*a" -hdr ":scheme" "http" -hdr ":method" "GET" + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + +client c1 { + stream 1 { + txreq -noadd -hdr ":authority" "foo.com" -hdr ":path" "*" -hdr ":scheme" "http" -hdr ":method" "GET" + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + +client c1 { + stream 1 { + txreq -noadd -hdr ":authority" "foo.com" -hdr ":path" "*" -hdr ":scheme" "http" -hdr ":method" "OPTIONS" + rxresp + expect resp.status == 200 + } -run +} -run + +client c1 { + stream 1 { + txreq -noadd -hdr ":authority" "foo.com" -hdr ":path" "*" -hdr ":scheme" "http" -hdr ":method" "OPTIONs" + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + +client c1 { + stream 1 { + txreq -noadd -hdr ":authority" "foo.com" -hdr ":path" "*" -hdr ":scheme" "http" -hdr ":method" "OPTIONSx" + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:09 +0000 (UTC) Subject: [6.0] e32ab13b8 h2: Relax "no //" URLs requirement Message-ID: <20240404143309.6F40F1038E9@lists.varnish-cache.org> commit e32ab13b8c1a63fc55c4e1749a6cd7aae44cee8d Author: Dag Haavi Finstad Date: Mon Apr 3 10:26:51 2023 +0200 h2: Relax "no //" URLs requirement This requirement was dropped in the updated rfc 9113. Fixes: #3911 diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 1e46c4b8d..57ca82390 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -134,10 +134,9 @@ h2h_addhdr(struct h2h_decode *d, struct http *hp, char *b, size_t namelen, n = HTTP_HDR_URL; disallow_empty = 1; - // rfc7540,l,3060,3071 - if (((len > 0 && *b != '/') || - (len > 1 && *(b+1) == '/')) && - (strncmp(b, "*", len) != 0)) { + // rfc9113,l,2693,2705 + if (len > 0 && *b != '/' && + strncmp(b, "*", len) != 0) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal :path pseudo-header %.*s", (int)len, b); diff --git a/bin/varnishtest/tests/a02027.vtc b/bin/varnishtest/tests/a02027.vtc index ff34b0071..e9dcf6619 100644 --- a/bin/varnishtest/tests/a02027.vtc +++ b/bin/varnishtest/tests/a02027.vtc @@ -22,8 +22,8 @@ client c1 { client c1 { stream 1 { txreq -noadd -hdr ":authority" "foo.com" -hdr ":path" "//foo" -hdr ":scheme" "http" -hdr ":method" "GET" - rxrst - expect rst.err == PROTOCOL_ERROR + rxresp + expect resp.status == 200 } -run } -run From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:09 +0000 (UTC) Subject: [6.0] 9cb03fe7e Redo H2 field validation Message-ID: <20240404143309.9C9D61038F8@lists.varnish-cache.org> commit 9cb03fe7e5a98a7aecdb6d11d6656bb3c7a509e4 Author: Poul-Henning Kamp Date: Tue Aug 22 08:41:21 2023 +0000 Redo H2 field validation Fixes #3952 diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 57ca82390..016e78c11 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -37,6 +37,7 @@ #include "http2/cache_http2.h" #include "vct.h" +// rfc9113,l,2493,2528 static h2_error h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) { @@ -46,46 +47,80 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) AN(b); assert(namelen >= 2); /* 2 chars from the ': ' that we added */ assert(namelen <= len); + assert(b[namelen - 2] == ':'); + assert(b[namelen - 1] == ' '); if (namelen == 2) { VSLb(hp->vsl, SLT_BogoHeader, "Empty name"); return (H2SE_PROTOCOL_ERROR); } - for (p = b; p < b + len; p++) { - if (p < b + (namelen - 2)) { - /* Check valid name characters */ - if (p == b && *p == ':') - continue; /* pseudo-header */ + // VSLb(hp->vsl, SLT_Debug, "CHDR [%.*s] [%.*s]", + // (int)namelen, b, (int)(len - namelen), b + namelen); + + int state = 0; + for (p = b; p < b + namelen - 2; p++) { + switch(state) { + case 0: /* First char of field */ + state = 1; + if (*p == ':') + break; + /* FALL_THROUGH */ + case 1: /* field name */ + if (*p <= 0x20 || *p >= 0x7f) { + VSLb(hp->vsl, SLT_BogoHeader, + "Illegal field header name (control): %.*s", + (int)(len > 20 ? 20 : len), b); + return (H2SE_PROTOCOL_ERROR); + } if (isupper(*p)) { VSLb(hp->vsl, SLT_BogoHeader, - "Illegal header name (upper-case): %.*s", + "Illegal field header name (upper-case): %.*s", (int)(len > 20 ? 20 : len), b); return (H2SE_PROTOCOL_ERROR); } - if (vct_istchar(*p)) { - /* XXX: vct should have a proper class for - this avoiding two checks */ - continue; + if (!vct_istchar(*p) || *p == ':') { + VSLb(hp->vsl, SLT_BogoHeader, + "Illegal field header name (non-token): %.*s", + (int)(len > 20 ? 20 : len), b); + return (H2SE_PROTOCOL_ERROR); } - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal header name: %.*s", - (int)(len > 20 ? 20 : len), b); - return (H2SE_PROTOCOL_ERROR); - } else if (p < b + namelen) { - /* ': ' added by us */ - assert(*p == ':' || *p == ' '); - } else { - /* Check valid value characters */ - if (!vct_isctl(*p) || vct_issp(*p)) - continue; - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal header value: %.*s", - (int)(len > 20 ? 20 : len), b); - return (H2SE_PROTOCOL_ERROR); + break; + default: + WRONG("http2 field name validation state"); } } + state = 2; + for (p = b + namelen; p < b + len; p++) { + switch(state) { + case 2: /* First char of field */ + if (*p == ' ' || *p == 0x09) { + VSLb(hp->vsl, SLT_BogoHeader, + "Illegal field value start %.*s", + (int)(len > 20 ? 20 : len), b); + return (H2SE_PROTOCOL_ERROR); + } + state = 3; + /* FALL_THROUGH */ + case 3: /* field value character */ + if (*p != 0x09 && (*p < 0x20 || *p == 0x7f)) { + VSLb(hp->vsl, SLT_BogoHeader, + "Illegal field value (control) %.*s", + (int)(len > 20 ? 20 : len), b); + return (H2SE_PROTOCOL_ERROR); + } + break; + default: + WRONG("http2 field value validation state"); + } + } + if (state == 3 && b[len - 1] <= 0x20) { + VSLb(hp->vsl, SLT_BogoHeader, + "Illegal val (end) %.*s", + (int)(len > 20 ? 20 : len), b); + return (H2SE_PROTOCOL_ERROR); + } return (0); } @@ -281,6 +316,7 @@ h2h_decode_fini(const struct h2_sess *h2) * block. This is a connection level error. * * H2E_PROTOCOL_ERROR: Malformed header or duplicate pseudo-header. + * Violation of field name/value charsets */ h2_error h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) diff --git a/bin/varnishtest/tests/t02023.vtc b/bin/varnishtest/tests/t02023.vtc index cfd843da3..59c7fe5d7 100644 --- a/bin/varnishtest/tests/t02023.vtc +++ b/bin/varnishtest/tests/t02023.vtc @@ -46,3 +46,47 @@ client c1 { rxrst } -run } -run + +varnish v1 -vsl_catchup + +client c1 { + stream 1 { + txreq -hdr "fo o" " bar" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -hdr "foo" " " + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -hdr ":foo" "bar" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -hdr "foo" "b\x0car" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -hdr "f o" "bar" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -hdr "f: o" "bar" + rxrst + } -run +} -run From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:09 +0000 (UTC) Subject: [6.0] f1b718441 hpack: Turn header validation state into an enum Message-ID: <20240404143309.CB2F3103919@lists.varnish-cache.org> commit f1b71844129eb072c47cd8d1f4f43897d1d2ba9f Author: Walid Boudebouda Date: Fri Sep 8 16:55:18 2023 +0200 hpack: Turn header validation state into an enum Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 016e78c11..d90638c81 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -42,6 +42,12 @@ static h2_error h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) { const char *p; + enum { + FLD_NAME_FIRST, + FLD_NAME, + FLD_VALUE_FIRST, + FLD_VALUE + } state; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); AN(b); @@ -58,15 +64,15 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) // VSLb(hp->vsl, SLT_Debug, "CHDR [%.*s] [%.*s]", // (int)namelen, b, (int)(len - namelen), b + namelen); - int state = 0; + state = FLD_NAME_FIRST; for (p = b; p < b + namelen - 2; p++) { switch(state) { - case 0: /* First char of field */ - state = 1; + case FLD_NAME_FIRST: + state = FLD_NAME; if (*p == ':') break; /* FALL_THROUGH */ - case 1: /* field name */ + case FLD_NAME: if (*p <= 0x20 || *p >= 0x7f) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal field header name (control): %.*s", @@ -91,19 +97,19 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } } - state = 2; + state = FLD_VALUE_FIRST; for (p = b + namelen; p < b + len; p++) { switch(state) { - case 2: /* First char of field */ + case FLD_VALUE_FIRST: if (*p == ' ' || *p == 0x09) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal field value start %.*s", (int)(len > 20 ? 20 : len), b); return (H2SE_PROTOCOL_ERROR); } - state = 3; + state = FLD_VALUE; /* FALL_THROUGH */ - case 3: /* field value character */ + case FLD_VALUE: if (*p != 0x09 && (*p < 0x20 || *p == 0x7f)) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal field value (control) %.*s", @@ -115,7 +121,7 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) WRONG("http2 field value validation state"); } } - if (state == 3 && b[len - 1] <= 0x20) { + if (state == FLD_VALUE && b[len - 1] <= 0x20) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal val (end) %.*s", (int)(len > 20 ? 20 : len), b); From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:09 +0000 (UTC) Subject: [6.0] ff52f481e hpack: Check illegal header blanks with vct_issp() Message-ID: <20240404143310.06D7D103940@lists.varnish-cache.org> commit ff52f481ee9c33daae1fe3956ba6fedd935ed0da Author: Walid Boudebouda Date: Fri Sep 8 17:25:06 2023 +0200 hpack: Check illegal header blanks with vct_issp() Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index d90638c81..529bc31eb 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -101,7 +101,7 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) for (p = b + namelen; p < b + len; p++) { switch(state) { case FLD_VALUE_FIRST: - if (*p == ' ' || *p == 0x09) { + if (vct_issp(*p)) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal field value start %.*s", (int)(len > 20 ? 20 : len), b); @@ -121,7 +121,7 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) WRONG("http2 field value validation state"); } } - if (state == FLD_VALUE && b[len - 1] <= 0x20) { + if (state == FLD_VALUE && vct_issp(b[len - 1])) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal val (end) %.*s", (int)(len > 20 ? 20 : len), b); From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:10 +0000 (UTC) Subject: [6.0] 9c85dcbb3 hpack: Validate header values with vct_ishdrval() Message-ID: <20240404143310.2DA1E103953@lists.varnish-cache.org> commit 9c85dcbb31caf3a01a22b4fb148bd9b081839eae Author: Walid Boudebouda Date: Fri Sep 8 17:37:26 2023 +0200 hpack: Validate header values with vct_ishdrval() Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 529bc31eb..4a1140291 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -110,9 +110,9 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) state = FLD_VALUE; /* FALL_THROUGH */ case FLD_VALUE: - if (*p != 0x09 && (*p < 0x20 || *p == 0x7f)) { + if (!vct_ishdrval(*p)) { VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field value (control) %.*s", + "Illegal field value %.*s", (int)(len > 20 ? 20 : len), b); return (H2SE_PROTOCOL_ERROR); } From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:10 +0000 (UTC) Subject: [6.0] e301d5fc9 hpack: Remove redundant/incorrect header validation Message-ID: <20240404143310.4651E10395B@lists.varnish-cache.org> commit e301d5fc918a834499e2bd3eddc82e282b951219 Author: Walid Boudebouda Date: Fri Sep 8 17:13:19 2023 +0200 hpack: Remove redundant/incorrect header validation Control characters will be caught by vct_ishdrval() anyways, but this condition would also reject allowed obs-text non-ASCII characters. Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 4a1140291..4d3a8819c 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -73,12 +73,6 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) break; /* FALL_THROUGH */ case FLD_NAME: - if (*p <= 0x20 || *p >= 0x7f) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field header name (control): %.*s", - (int)(len > 20 ? 20 : len), b); - return (H2SE_PROTOCOL_ERROR); - } if (isupper(*p)) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal field header name (upper-case): %.*s", From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:10 +0000 (UTC) Subject: [6.0] ef7178662 http2_hpack: Reorganize header addition for clarity Message-ID: <20240404143310.7356D103972@lists.varnish-cache.org> commit ef7178662d478584c21788f0e63c144c5c52efc0 Author: Dridi Boukelmoune Date: Wed Mar 27 16:09:19 2024 +0100 http2_hpack: Reorganize header addition for clarity Instead of passing both a decoder and individual decoder fields, the signature for h2h_addhdr() changed to only take the decoder. The order of parameters is destination first, then the source following the calling conventon of functions like memcpy(). Internally the function is reorganized with a bunch of txt variables to keep track of the header being added, its name and value. In addition to clarity, this also helps improve safety and correctness. For example the :authority pseudo-header name is erased in place to turn it into a regular host header, but having a dedicated txt for the header name allows its preservation. diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 4d3a8819c..739cb0e80 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -37,6 +37,18 @@ #include "http2/cache_http2.h" #include "vct.h" +static void +h2h_assert_ready(struct h2h_decode *d) +{ + + CHECK_OBJ_NOTNULL(d, H2H_DECODE_MAGIC); + AN(d->out); + assert(d->namelen >= 2); /* 2 chars from the ": " that we added */ + assert(d->namelen <= d->out_u); + assert(d->out[d->namelen - 2] == ':'); + assert(d->out[d->namelen - 1] == ' '); +} + // rfc9113,l,2493,2528 static h2_error h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) @@ -125,132 +137,130 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } static h2_error -h2h_addhdr(struct h2h_decode *d, struct http *hp, char *b, size_t namelen, - size_t len) +h2h_addhdr(struct http *hp, struct h2h_decode *d) { /* XXX: This might belong in cache/cache_http.c */ - const char *b0; + txt hdr, nm, val; int disallow_empty; + const char *p; unsigned n; - char *p; - unsigned u; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - AN(b); - assert(namelen >= 2); /* 2 chars from the ': ' that we added */ - assert(namelen <= len); + h2h_assert_ready(d); + + /* Assume hdr is by default a regular header from what we decoded. */ + hdr.b = d->out; + hdr.e = hdr.b + d->out_u; + n = hp->nhd; + + /* nm and val are separated by ": " */ + nm.b = hdr.b; + nm.e = nm.b + d->namelen - 2; + val.b = nm.e + 2; + val.e = hdr.e; disallow_empty = 0; - if (len > UINT_MAX) { /* XXX: cache_param max header size */ - VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", b); + if (Tlen(hdr) > UINT_MAX) { /* XXX: cache_param max header size */ + VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", hdr.b); return (H2SE_ENHANCE_YOUR_CALM); } - b0 = b; - if (b[0] == ':') { + if (*nm.b == ':') { /* Match H/2 pseudo headers */ /* XXX: Should probably have some include tbl for pseudo-headers */ - if (!strncmp(b, ":method: ", namelen)) { - b += namelen; - len -= namelen; + if (!Tstrcmp(nm, ":method")) { + hdr.b = val.b; n = HTTP_HDR_METHOD; disallow_empty = 1; - /* First field cannot contain SP or CTL */ - for (p = b, u = 0; u < len; p++, u++) { - if (vct_issp(*p) || vct_isctl(*p)) + /* Check HTTP token */ + for (p = hdr.b; p < hdr.e; p++) { + if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); } - } else if (!strncmp(b, ":path: ", namelen)) { - b += namelen; - len -= namelen; + } else if (!Tstrcmp(nm, ":path")) { + hdr.b = val.b; n = HTTP_HDR_URL; disallow_empty = 1; // rfc9113,l,2693,2705 - if (len > 0 && *b != '/' && - strncmp(b, "*", len) != 0) { + if (Tlen(val) > 0 && *val.b != '/' && + Tstrcmp(val, "*")) { VSLb(hp->vsl, SLT_BogoHeader, "Illegal :path pseudo-header %.*s", - (int)len, b); + (int)Tlen(val), val.b); return (H2SE_PROTOCOL_ERROR); } - /* Second field cannot contain LWS or CTL */ - for (p = b, u = 0; u < len; p++, u++) { + /* Path cannot contain LWS or CTL */ + for (p = hdr.b; p < hdr.e; p++) { if (vct_islws(*p) || vct_isctl(*p)) return (H2SE_PROTOCOL_ERROR); } - } else if (!strncmp(b, ":scheme: ", namelen)) { + } else if (!Tstrcmp(nm, ":scheme")) { /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ if (d->has_scheme) { VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header %.*s%.*s", - (int)namelen, b0, - (int)(len > 20 ? 20 : len), b); + "Duplicate pseudo-header :scheme: %.*s", + vmin_t(int, Tlen(val), 20), val.b); return (H2SE_PROTOCOL_ERROR); } - b++; - len-=1; - n = hp->nhd; + hdr.b++; d->has_scheme = 1; + disallow_empty = 1; - for (p = b + namelen, u = 0; u < len-namelen; - p++, u++) { - if (vct_issp(*p) || vct_isctl(*p)) + /* Check HTTP token */ + for (p = val.b; p < val.e; p++) { + if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); } - - if (!u) - return (H2SE_PROTOCOL_ERROR); - } else if (!strncmp(b, ":authority: ", namelen)) { - b+=6; - len-=6; - memcpy(b, "host", 4); - n = hp->nhd; + } else if (!Tstrcmp(nm, ":authority")) { + /* NB: we inject "host" in place of "rity" for + * the ":authority" pseudo-header. + */ + memcpy(d->out + 6, "host", 4); + hdr.b += 6; + nm = Tstr(":authority"); /* preserve original */ } else { /* Unknown pseudo-header */ VSLb(hp->vsl, SLT_BogoHeader, "Unknown pseudo-header: %.*s", - (int)(len > 20 ? 20 : len), b); + vmin_t(int, Tlen(hdr), 20), hdr.b); return (H2SE_PROTOCOL_ERROR); // rfc7540,l,2990,2992 } - } else - n = hp->nhd; + } + + if (disallow_empty && Tlen(val) == 0) { + VSLb(hp->vsl, SLT_BogoHeader, + "Empty pseudo-header %.*s", + (int)Tlen(nm), nm.b); + return (H2SE_PROTOCOL_ERROR); + } if (n < HTTP_HDR_FIRST) { - /* Check for duplicate pseudo-header */ if (hp->hd[n].b != NULL) { VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header %.*s%.*s", - (int)namelen, b0, (int)(len > 20 ? 20 : len), b); + "Duplicate pseudo-header %.*s", + (int)Tlen(nm), nm.b); return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 } } else { /* Check for space in struct http */ if (n >= hp->shd) { - VSLb(hp->vsl, SLT_LostHeader, "Too many headers: %.*s", - (int)(len > 20 ? 20 : len), b); + VSLb(hp->vsl, SLT_LostHeader, + "Too many headers: %.*s", + vmin_t(int, Tlen(hdr), 20), hdr.b); return (H2SE_ENHANCE_YOUR_CALM); } hp->nhd++; } - hp->hd[n].b = b; - hp->hd[n].e = b + len; - - if (disallow_empty && !Tlen(hp->hd[n])) { - VSLb(hp->vsl, SLT_BogoHeader, - "Empty pseudo-header %.*s", - (int)namelen, b0); - return (H2SE_PROTOCOL_ERROR); - } - + hp->hd[n] = hdr; return (0); } @@ -388,8 +398,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out_u); if (d->error) break; - d->error = h2h_addhdr(d, hp, d->out, - d->namelen, d->out_u); + d->error = h2h_addhdr(hp, d); if (d->error) break; d->out[d->out_u++] = '\0'; /* Zero guard */ From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:10 +0000 (UTC) Subject: [6.0] 2069001c4 http2_hpack: Refuse multiple :authority pseudo headers Message-ID: <20240404143310.8AF06103985@lists.varnish-cache.org> commit 2069001c4a003489530f14746f4fbdd91a0d585e Author: Dridi Boukelmoune Date: Wed Mar 27 16:17:08 2024 +0100 http2_hpack: Refuse multiple :authority pseudo headers It became explicit in rfc9113: > The same pseudo-header field name MUST NOT appear more than once in a > field block. While at it, the duplicate pseudo-header error can be consolidated in a single location instead of adding one more branch. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 53339dbe5..26cba9055 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -226,6 +226,7 @@ struct h2h_decode { unsigned magic; #define H2H_DECODE_MAGIC 0xd092bde4 + unsigned has_authority:1; unsigned has_scheme:1; h2_error error; enum vhd_ret_e vhd_ret; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 739cb0e80..14009502e 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -143,7 +143,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) txt hdr, nm, val; int disallow_empty; const char *p; - unsigned n; + unsigned n, has_dup; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); h2h_assert_ready(d); @@ -160,6 +160,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) val.e = hdr.e; disallow_empty = 0; + has_dup = 0; if (Tlen(hdr) > UINT_MAX) { /* XXX: cache_param max header size */ VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", hdr.b); @@ -203,14 +204,8 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ - if (d->has_scheme) { - VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header :scheme: %.*s", - vmin_t(int, Tlen(val), 20), val.b); - return (H2SE_PROTOCOL_ERROR); - } - hdr.b++; + has_dup = d->has_scheme; d->has_scheme = 1; disallow_empty = 1; @@ -226,6 +221,8 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) memcpy(d->out + 6, "host", 4); hdr.b += 6; nm = Tstr(":authority"); /* preserve original */ + has_dup = d->has_authority; + d->has_authority = 1; } else { /* Unknown pseudo-header */ VSLb(hp->vsl, SLT_BogoHeader, @@ -242,14 +239,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) return (H2SE_PROTOCOL_ERROR); } - if (n < HTTP_HDR_FIRST) { - if (hp->hd[n].b != NULL) { - VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header %.*s", - (int)Tlen(nm), nm.b); - return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 - } - } else { + if (n >= HTTP_HDR_FIRST) { /* Check for space in struct http */ if (n >= hp->shd) { VSLb(hp->vsl, SLT_LostHeader, @@ -258,6 +248,15 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) return (H2SE_ENHANCE_YOUR_CALM); } hp->nhd++; + AZ(hp->hd[n].b); + } + + if (has_dup || hp->hd[n].b != NULL) { + assert(nm.b[0] == ':'); + VSLb(hp->vsl, SLT_BogoHeader, + "Duplicate pseudo-header %.*s", + (int)Tlen(nm), nm.b); + return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 } hp->hd[n] = hdr; diff --git a/bin/varnishtest/tests/r02351.vtc b/bin/varnishtest/tests/r02351.vtc index d2ee19af8..6fdee96d5 100644 --- a/bin/varnishtest/tests/r02351.vtc +++ b/bin/varnishtest/tests/r02351.vtc @@ -1,4 +1,4 @@ -varnishtest "#2351: :path/:method error handling" +varnishtest "#2351: h2 pseudo-headers error handling" server s1 { rxreq @@ -39,6 +39,16 @@ client c1 { } -run } -run +client c2 { + # Duplicate :authority + stream next { + txreq -noadd -hdr :path / -hdr :method GET -hdr :scheme http \ + -hdr :authority example.com -hdr :authority example.org + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + varnish v1 -expect MEMPOOL.req0.live == 0 varnish v1 -expect MEMPOOL.req1.live == 0 varnish v1 -expect MEMPOOL.sess0.live == 0 From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:10 +0000 (UTC) Subject: [6.0] bc0bb5b99 http2_hpack: Remove one level of nesting Message-ID: <20240404143310.AD5FB103991@lists.varnish-cache.org> commit bc0bb5b99fd8d42adeb957cb69eaed765fb39a80 Author: Dridi Boukelmoune Date: Wed Mar 27 16:28:18 2024 +0100 http2_hpack: Remove one level of nesting Better diff with the --ignore-all-space option. diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 14009502e..264226201 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -167,69 +167,64 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) return (H2SE_ENHANCE_YOUR_CALM); } - if (*nm.b == ':') { - /* Match H/2 pseudo headers */ - /* XXX: Should probably have some include tbl for - pseudo-headers */ - if (!Tstrcmp(nm, ":method")) { - hdr.b = val.b; - n = HTTP_HDR_METHOD; - disallow_empty = 1; - - /* Check HTTP token */ - for (p = hdr.b; p < hdr.e; p++) { - if (!vct_istchar(*p)) - return (H2SE_PROTOCOL_ERROR); - } - } else if (!Tstrcmp(nm, ":path")) { - hdr.b = val.b; - n = HTTP_HDR_URL; - disallow_empty = 1; - - // rfc9113,l,2693,2705 - if (Tlen(val) > 0 && *val.b != '/' && - Tstrcmp(val, "*")) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal :path pseudo-header %.*s", - (int)Tlen(val), val.b); + /* Match H/2 pseudo headers */ + /* XXX: Should probably have some include tbl for pseudo-headers */ + if (!Tstrcmp(nm, ":method")) { + hdr.b = val.b; + n = HTTP_HDR_METHOD; + disallow_empty = 1; + + /* Check HTTP token */ + for (p = hdr.b; p < hdr.e; p++) { + if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); - } + } + } else if (!Tstrcmp(nm, ":path")) { + hdr.b = val.b; + n = HTTP_HDR_URL; + disallow_empty = 1; - /* Path cannot contain LWS or CTL */ - for (p = hdr.b; p < hdr.e; p++) { - if (vct_islws(*p) || vct_isctl(*p)) - return (H2SE_PROTOCOL_ERROR); - } - } else if (!Tstrcmp(nm, ":scheme")) { - /* XXX: What to do about this one? (typically - "http" or "https"). For now set it as a normal - header, stripping the first ':'. */ - hdr.b++; - has_dup = d->has_scheme; - d->has_scheme = 1; - disallow_empty = 1; - - /* Check HTTP token */ - for (p = val.b; p < val.e; p++) { - if (!vct_istchar(*p)) - return (H2SE_PROTOCOL_ERROR); - } - } else if (!Tstrcmp(nm, ":authority")) { - /* NB: we inject "host" in place of "rity" for - * the ":authority" pseudo-header. - */ - memcpy(d->out + 6, "host", 4); - hdr.b += 6; - nm = Tstr(":authority"); /* preserve original */ - has_dup = d->has_authority; - d->has_authority = 1; - } else { - /* Unknown pseudo-header */ + // rfc9113,l,2693,2705 + if (Tlen(val) > 0 && val.b[0] != '/' && Tstrcmp(val, "*")) { VSLb(hp->vsl, SLT_BogoHeader, - "Unknown pseudo-header: %.*s", - vmin_t(int, Tlen(hdr), 20), hdr.b); - return (H2SE_PROTOCOL_ERROR); // rfc7540,l,2990,2992 + "Illegal :path pseudo-header %.*s", + (int)Tlen(val), val.b); + return (H2SE_PROTOCOL_ERROR); } + + /* Path cannot contain LWS or CTL */ + for (p = hdr.b; p < hdr.e; p++) { + if (vct_islws(*p) || vct_isctl(*p)) + return (H2SE_PROTOCOL_ERROR); + } + } else if (!Tstrcmp(nm, ":scheme")) { + /* XXX: What to do about this one? (typically + "http" or "https"). For now set it as a normal + header, stripping the first ':'. */ + hdr.b++; + has_dup = d->has_scheme; + d->has_scheme = 1; + disallow_empty = 1; + + /* Check HTTP token */ + for (p = val.b; p < val.e; p++) { + if (!vct_istchar(*p)) + return (H2SE_PROTOCOL_ERROR); + } + } else if (!Tstrcmp(nm, ":authority")) { + /* NB: we inject "host" in place of "rity" for + * the ":authority" pseudo-header. + */ + memcpy(d->out + 6, "host", 4); + hdr.b += 6; + nm = Tstr(":authority"); /* preserve original */ + has_dup = d->has_authority; + d->has_authority = 1; + } else if (nm.b[0] == ':') { + VSLb(hp->vsl, SLT_BogoHeader, + "Unknown pseudo-header: %.*s", + vmin_t(int, Tlen(hdr), 20), hdr.b); + return (H2SE_PROTOCOL_ERROR); // rfc7540,l,2990,2992 } if (disallow_empty && Tlen(val) == 0) { From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:10 +0000 (UTC) Subject: [6.0] d0d092ede http2_hpack: Also rewrite h2h_checkhdr() for clarity Message-ID: <20240404143310.C662410399E@lists.varnish-cache.org> commit d0d092ede56a98121177f7e81ef3b24a87475f14 Author: Dridi Boukelmoune Date: Thu Mar 28 14:13:36 2024 +0100 http2_hpack: Also rewrite h2h_checkhdr() for clarity It does a first pass on header names and values, and only logs errors, so the signature is updated accordingly and the call site is moved into h2h_addhdr(). Conflicts: bin/varnishd/http2/cache_http2_hpack.c A warning triggered despite the presence of FALL_THROUGH. diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 264226201..717d93fe4 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -51,9 +51,10 @@ h2h_assert_ready(struct h2h_decode *d) // rfc9113,l,2493,2528 static h2_error -h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) +h2h_checkhdr(struct vsl_log *vsl, txt nm, txt val) { const char *p; + int l; enum { FLD_NAME_FIRST, FLD_NAME, @@ -61,40 +62,34 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) FLD_VALUE } state; - CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - AN(b); - assert(namelen >= 2); /* 2 chars from the ': ' that we added */ - assert(namelen <= len); - assert(b[namelen - 2] == ':'); - assert(b[namelen - 1] == ' '); - - if (namelen == 2) { - VSLb(hp->vsl, SLT_BogoHeader, "Empty name"); + if (Tlen(nm) == 0) { + VSLb(vsl, SLT_BogoHeader, "Empty name"); return (H2SE_PROTOCOL_ERROR); } - // VSLb(hp->vsl, SLT_Debug, "CHDR [%.*s] [%.*s]", - // (int)namelen, b, (int)(len - namelen), b + namelen); + // VSLb(vsl, SLT_Debug, "CHDR [%.*s] [%.*s]", + // (int)Tlen(nm), nm.b, (int)Tlen(val), val.b); + l = vmin_t(int, Tlen(nm) + 2 + Tlen(val), 20); state = FLD_NAME_FIRST; - for (p = b; p < b + namelen - 2; p++) { + for (p = nm.b; p < nm.e; p++) { switch(state) { case FLD_NAME_FIRST: state = FLD_NAME; if (*p == ':') break; - /* FALL_THROUGH */ + /* FALLTHROUGH */ case FLD_NAME: if (isupper(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, + VSLb(vsl, SLT_BogoHeader, "Illegal field header name (upper-case): %.*s", - (int)(len > 20 ? 20 : len), b); + l, nm.b); return (H2SE_PROTOCOL_ERROR); } if (!vct_istchar(*p) || *p == ':') { - VSLb(hp->vsl, SLT_BogoHeader, + VSLb(vsl, SLT_BogoHeader, "Illegal field header name (non-token): %.*s", - (int)(len > 20 ? 20 : len), b); + l, nm.b); return (H2SE_PROTOCOL_ERROR); } break; @@ -104,22 +99,20 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } state = FLD_VALUE_FIRST; - for (p = b + namelen; p < b + len; p++) { + for (p = val.b; p < val.e; p++) { switch(state) { case FLD_VALUE_FIRST: if (vct_issp(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field value start %.*s", - (int)(len > 20 ? 20 : len), b); + VSLb(vsl, SLT_BogoHeader, + "Illegal field value start %.*s", l, nm.b); return (H2SE_PROTOCOL_ERROR); } state = FLD_VALUE; - /* FALL_THROUGH */ + /* FALLTHROUGH */ case FLD_VALUE: if (!vct_ishdrval(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal field value %.*s", - (int)(len > 20 ? 20 : len), b); + VSLb(vsl, SLT_BogoHeader, + "Illegal field value %.*s", l, nm.b); return (H2SE_PROTOCOL_ERROR); } break; @@ -127,10 +120,9 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) WRONG("http2 field value validation state"); } } - if (state == FLD_VALUE && vct_issp(b[len - 1])) { - VSLb(hp->vsl, SLT_BogoHeader, - "Illegal val (end) %.*s", - (int)(len > 20 ? 20 : len), b); + if (state == FLD_VALUE && vct_issp(val.e[-1])) { + VSLb(vsl, SLT_BogoHeader, + "Illegal field value (end) %.*s", l, nm.b); return (H2SE_PROTOCOL_ERROR); } return (0); @@ -144,6 +136,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) int disallow_empty; const char *p; unsigned n, has_dup; + h2_error err; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); h2h_assert_ready(d); @@ -159,6 +152,10 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) val.b = nm.e + 2; val.e = hdr.e; + err = h2h_checkhdr(hp->vsl, nm, val); + if (err != NULL) + return (err); + disallow_empty = 0; has_dup = 0; @@ -388,10 +385,6 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->error = H2SE_ENHANCE_YOUR_CALM; break; } - d->error = h2h_checkhdr(hp, d->out, d->namelen, - d->out_u); - if (d->error) - break; d->error = h2h_addhdr(hp, d); if (d->error) break; From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:10 +0000 (UTC) Subject: [6.0] 8cee9a0b1 vtc: Coverage for initial space in :scheme: pseudo-header Message-ID: <20240404143310.E40AF1039AC@lists.varnish-cache.org> commit 8cee9a0b1b474dcbacdd199b9b21da422bd698ff Author: Dridi Boukelmoune Date: Fri Mar 29 16:48:25 2024 +0100 vtc: Coverage for initial space in :scheme: pseudo-header diff --git a/bin/varnishtest/tests/t02024.vtc b/bin/varnishtest/tests/t02024.vtc index 0d0a1abc5..35820d18f 100644 --- a/bin/varnishtest/tests/t02024.vtc +++ b/bin/varnishtest/tests/t02024.vtc @@ -40,6 +40,13 @@ client c1 { } -run } -run +client c1 { + stream 1 { + txreq -scheme " space" + rxrst + } -run +} -run + client c1 { stream 1 { txreq -req " " From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:11 +0000 (UTC) Subject: [6.0] 745f2cdb2 http2_hpack: Enforce http_req_hdr_len limit Message-ID: <20240404143311.1DE111039C4@lists.varnish-cache.org> commit 745f2cdb25b6ede8ad8f332031dab555de39a7d7 Author: Dridi Boukelmoune Date: Thu Mar 28 15:21:01 2024 +0100 http2_hpack: Enforce http_req_hdr_len limit Refs #3709 diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 717d93fe4..22eaa61f1 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -159,7 +159,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) disallow_empty = 0; has_dup = 0; - if (Tlen(hdr) > UINT_MAX) { /* XXX: cache_param max header size */ + if (Tlen(hdr) > cache_param->http_req_hdr_len) { VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", hdr.b); return (H2SE_ENHANCE_YOUR_CALM); } diff --git a/bin/varnishtest/tests/r03709.vtc b/bin/varnishtest/tests/r03709.vtc new file mode 100644 index 000000000..7439efba3 --- /dev/null +++ b/bin/varnishtest/tests/r03709.vtc @@ -0,0 +1,21 @@ +varnishtest "h2 req limits" + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set http_req_hdr_len 40b" +varnish v1 -vcl { + backend be none; +} -start + +logexpect l1 -v v1 -g raw -q BogoHeader { + expect 0 1001 BogoHeader "Header too large: :path" +} -start + +client c1 { + stream next { + txreq -url ${string,repeat,4,/123456789} + rxrst + expect rst.err == ENHANCE_YOUR_CALM + } -run +} -run + +logexpect l1 -wait From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:11 +0000 (UTC) Subject: [6.0] fc857a6c5 http2: New H2_ERROR_MATCH() helper macro Message-ID: <20240404143311.395E01039D9@lists.varnish-cache.org> commit fc857a6c599b7a2b4aa8745ba30885ee8b6430e8 Author: Dridi Boukelmoune Date: Thu Mar 28 15:56:21 2024 +0100 http2: New H2_ERROR_MATCH() helper macro diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 26cba9055..00fbe7c6b 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -48,6 +48,9 @@ struct h2_error_s { typedef const struct h2_error_s *h2_error; +#define H2_ERROR_MATCH(err, target) \ + ((err) != NULL && (err)->val == (target)->val) + #define H2_CUSTOM_ERRORS #define H2EC1(U,v,g,r,d) extern const struct h2_error_s H2CE_##U[1]; #define H2EC2(U,v,g,r,d) extern const struct h2_error_s H2SE_##U[1]; diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index c1e03a5dc..6ae45d136 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -448,6 +448,6 @@ H2_Send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, h2_send(wrk, r2, ftyp, flags, len, ptr, counter); h2e = h2_errcheck(r2, r2->h2sess); - if (h2e != NULL && h2e->val == H2SE_CANCEL->val) + if (H2_ERROR_MATCH(h2e, H2SE_CANCEL)) H2_Send_RST(wrk, r2->h2sess, r2, r2->stream, h2e); } From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:11 +0000 (UTC) Subject: [6.0] b4a5f99d0 http2_hpack: New custom h2 error for http_req_size Message-ID: <20240404143311.576C71039EB@lists.varnish-cache.org> commit b4a5f99d03520467898619d67f61fd571916c059 Author: Dridi Boukelmoune Date: Thu Mar 28 16:02:53 2024 +0100 http2_hpack: New custom h2 error for http_req_size diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 22eaa61f1..25fc0be7d 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -338,7 +338,8 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) /* Only H2E_ENHANCE_YOUR_CALM indicates that we should continue processing. Other errors should have been returned and handled by the caller. */ - assert(d->error == 0 || d->error == H2SE_ENHANCE_YOUR_CALM); + if (d->error != NULL) + assert(H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)); while (1) { AN(d->out); @@ -357,7 +358,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; } - if (d->error == H2SE_ENHANCE_YOUR_CALM) { + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { d->out_u = 0; assert(d->out_u < d->out_l); continue; @@ -369,7 +370,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) case VHD_NAME: assert(d->namelen == 0); if (d->out_l - d->out_u < 2) { - d->error = H2SE_ENHANCE_YOUR_CALM; + d->error = H2SE_REQ_SIZE; break; } d->out[d->out_u++] = ':'; @@ -382,7 +383,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) case VHD_VALUE: assert(d->namelen > 0); if (d->out_l - d->out_u < 1) { - d->error = H2SE_ENHANCE_YOUR_CALM; + d->error = H2SE_REQ_SIZE; break; } d->error = h2h_addhdr(hp, d); @@ -396,7 +397,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; case VHD_BUF: - d->error = H2SE_ENHANCE_YOUR_CALM; + d->error = H2SE_REQ_SIZE; break; default: @@ -404,7 +405,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; } - if (d->error == H2SE_ENHANCE_YOUR_CALM) { + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { d->out = d->reset; d->out_l = hp->ws->r - d->out; d->out_u = 0; @@ -413,7 +414,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) break; } - if (d->error == H2SE_ENHANCE_YOUR_CALM) + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) return (0); /* Stream error, delay reporting until h2h_decode_fini so that we can process the complete header block */ diff --git a/include/tbl/h2_error.h b/include/tbl/h2_error.h index 49436f861..2f1303e91 100644 --- a/include/tbl/h2_error.h +++ b/include/tbl/h2_error.h @@ -195,6 +195,15 @@ H2_ERROR( /* reason */ SC_BANKRUPT, /* descr */ "http/2 bankrupt connection" ) + +H2_ERROR( + /* name */ REQ_SIZE, + /* val */ 11, /* ENHANCE_YOUR_CALM */ + /* types */ 2, + /* goaway */ 0, + /* reason */ SC_NULL, + /* descr */ "HTTP/2 header list exceeded http_req_size" +) # undef H2_CUSTOM_ERRORS #endif From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:11 +0000 (UTC) Subject: [6.0] 39e5c9266 http2_hpack: Enforce http_req_size limit Message-ID: <20240404143311.76F13103A04@lists.varnish-cache.org> commit 39e5c9266e387cdb1f79a3ef33762eccd3827b87 Author: Dridi Boukelmoune Date: Thu Mar 28 16:08:46 2024 +0100 http2_hpack: Enforce http_req_size limit Refs #3709 Refs #3892 Conflicts: bin/varnishd/http2/cache_http2_hpack.c diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 25fc0be7d..1b476c9aa 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -267,7 +267,8 @@ h2h_decode_init(const struct h2_sess *h2) d = h2->decode; INIT_OBJ(d, H2H_DECODE_MAGIC); VHD_Init(d->vhd); - d->out_l = WS_ReserveAll(h2->new_req->http->ws); + d->out_l = WS_Reserve(h2->new_req->http->ws, + cache_param->http_req_size); /* * Can't do any work without any buffer * space. Require non-zero size. @@ -308,6 +309,10 @@ h2h_decode_fini(const struct h2_sess *h2) } else ret = d->error; d->magic = 0; + if (ret == H2SE_REQ_SIZE) { + VSLb(h2->new_req->http->vsl, SLT_LostHeader, + "Header list too large"); + } return (ret); } diff --git a/bin/varnishtest/tests/r03709.vtc b/bin/varnishtest/tests/r03709.vtc index 7439efba3..242afe2f1 100644 --- a/bin/varnishtest/tests/r03709.vtc +++ b/bin/varnishtest/tests/r03709.vtc @@ -2,17 +2,40 @@ varnishtest "h2 req limits" varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set http_req_hdr_len 40b" +varnish v1 -cliok "param.set http_req_size 512b" varnish v1 -vcl { backend be none; } -start -logexpect l1 -v v1 -g raw -q BogoHeader { +logexpect l1 -v v1 -g raw -q BogoHeader,LostHeader { expect 0 1001 BogoHeader "Header too large: :path" + expect 0 1002 LostHeader "Header list too large" } -start client c1 { stream next { - txreq -url ${string,repeat,4,/123456789} + txreq -url ${string,repeat,4,/123456789} \ + -hdr limit http_req_hdr_len + rxrst + expect rst.err == ENHANCE_YOUR_CALM + } -run + + stream next { + txreq -url "/http_req_size" \ + -hdr hdr1 ${string,repeat,3,/123456789} \ + -hdr hdr2 ${string,repeat,3,/123456789} \ + -hdr hdr3 ${string,repeat,3,/123456789} \ + -hdr hdr4 ${string,repeat,3,/123456789} \ + -hdr hdr5 ${string,repeat,3,/123456789} \ + -hdr hdr6 ${string,repeat,3,/123456789} \ + -hdr hdr7 ${string,repeat,3,/123456789} \ + -hdr hdr8 ${string,repeat,3,/123456789} \ + -hdr hdr9 ${string,repeat,3,/123456789} \ + -hdr hdr10 ${string,repeat,3,/123456789} \ + -hdr hdr11 ${string,repeat,3,/123456789} \ + -hdr hdr12 ${string,repeat,3,/123456789} \ + -hdr hdr13 ${string,repeat,3,/123456789} \ + -hdr hdr14 ${string,repeat,3,/123456789} rxrst expect rst.err == ENHANCE_YOUR_CALM } -run From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:11 +0000 (UTC) Subject: [6.0] f69a70e7d vtc: Add http_max_hdr coverage to r3709 Message-ID: <20240404143311.91E8A103A15@lists.varnish-cache.org> commit f69a70e7d31115110ef171a3627b082e75ce154e Author: Dridi Boukelmoune Date: Thu Mar 28 16:22:59 2024 +0100 vtc: Add http_max_hdr coverage to r3709 For the sole purpose of having these limits tested in a single place. diff --git a/bin/varnishtest/tests/r03709.vtc b/bin/varnishtest/tests/r03709.vtc index 242afe2f1..65038fcce 100644 --- a/bin/varnishtest/tests/r03709.vtc +++ b/bin/varnishtest/tests/r03709.vtc @@ -3,6 +3,7 @@ varnishtest "h2 req limits" varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set http_req_hdr_len 40b" varnish v1 -cliok "param.set http_req_size 512b" +varnish v1 -cliok "param.set http_max_hdr 32" varnish v1 -vcl { backend be none; } -start @@ -10,6 +11,7 @@ varnish v1 -vcl { logexpect l1 -v v1 -g raw -q BogoHeader,LostHeader { expect 0 1001 BogoHeader "Header too large: :path" expect 0 1002 LostHeader "Header list too large" + expect 0 1003 LostHeader "Too many headers" } -start client c1 { @@ -39,6 +41,41 @@ client c1 { rxrst expect rst.err == ENHANCE_YOUR_CALM } -run + + stream next { + txreq -url "/http_max_hdr" \ + -hdr hdr1 val1 \ + -hdr hdr2 val2 \ + -hdr hdr3 val3 \ + -hdr hdr4 val4 \ + -hdr hdr4 val4 \ + -hdr hdr5 val5 \ + -hdr hdr6 val6 \ + -hdr hdr7 val7 \ + -hdr hdr8 val8 \ + -hdr hdr9 val9 \ + -hdr hdr10 val10 \ + -hdr hdr11 val11 \ + -hdr hdr11 val11 \ + -hdr hdr11 val11 \ + -hdr hdr12 val12 \ + -hdr hdr13 val13 \ + -hdr hdr14 val14 \ + -hdr hdr15 val15 \ + -hdr hdr16 val16 \ + -hdr hdr17 val17 \ + -hdr hdr18 val18 \ + -hdr hdr19 val19 \ + -hdr hdr20 val20 \ + -hdr hdr20 val20 \ + -hdr hdr21 val21 \ + -hdr hdr22 val22 \ + -hdr hdr23 val23 \ + -hdr hdr24 val24 \ + -hdr hdr25 val25 + rxrst + expect rst.err == ENHANCE_YOUR_CALM + } -run } -run logexpect l1 -wait From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:11 +0000 (UTC) Subject: [6.0] 0e1e4ea94 param: Document mapping to HTTP/2 settings Message-ID: <20240404143311.B3011103A22@lists.varnish-cache.org> commit 0e1e4ea94e563eb2967dcdadc85eb850243e810a Author: Dridi Boukelmoune Date: Thu Mar 28 16:34:35 2024 +0100 param: Document mapping to HTTP/2 settings With the exception of h2_max_header_list_size that is not advertised as such despite being ent as part of the initial SETTINGS frame. The same parameter also sees its default and maximum values updated to 2^32-1. This is based on this sentence from rfc9113: > The initial value of this setting is unlimited. This aligns the h2_max_header_list_size parameter with the values set in h2_settings.h for MAX_HEADER_LIST_SIZE. Conflicts: include/tbl/params.h diff --git a/include/tbl/params.h b/include/tbl/params.h index 455a79ec1..5ec1ec821 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1816,6 +1816,12 @@ PARAM( /* func */ NULL ) +#define H2_SETTING_NAME(nm) "SETTINGS_" #nm +#define H2_SETTING_DESCR(nm) \ + "\n\nThe value of this parameter defines " H2_SETTING_NAME(nm) \ + " in the initial SETTINGS frame sent to the client when a new " \ + "HTTP2 session is established." + PARAM( /* name */ h2_header_table_size, /* typ */ bytes_u, @@ -1827,7 +1833,8 @@ PARAM( /* s-text */ "HTTP2 header table size.\n" "This is the size that will be used for the HPACK dynamic\n" - "decoding table.", + "decoding table." + H2_SETTING_DESCR(HEADER_TABLE_SIZE), /* l-text */ "", /* func */ NULL ) @@ -1843,7 +1850,8 @@ PARAM( /* s-text */ "HTTP2 Maximum number of concurrent streams.\n" "This is the number of requests that can be active\n" - "at the same time for a single HTTP2 connection.", + "at the same time for a single HTTP2 connection." + H2_SETTING_DESCR(MAX_CONCURRENT_STREAMS), /* l-text */ "", /* func */ NULL ) @@ -1861,7 +1869,8 @@ PARAM( /* units */ "bytes", /* flags */ 0, /* s-text */ - "HTTP2 initial flow control window size.", + "HTTP2 initial flow control window size." + H2_SETTING_DESCR(INITIAL_WINDOW_SIZE), /* l-text */ "", /* func */ NULL ) @@ -1875,7 +1884,8 @@ PARAM( /* units */ "bytes", /* flags */ 0, /* s-text */ - "HTTP2 maximum per frame payload size we are willing to accept.", + "HTTP2 maximum per frame payload size we are willing to accept." + H2_SETTING_DESCR(MAX_FRAME_SIZE), /* l-text */ "", /* func */ NULL ) @@ -1884,8 +1894,8 @@ PARAM( /* name */ h2_max_header_list_size, /* typ */ bytes_u, /* min */ "0b", - /* max */ NULL, - /* default */ "2147483647b", + /* max */ "4294967295b", + /* default */ "4294967295b", /* units */ "bytes", /* flags */ 0, /* s-text */ @@ -1894,6 +1904,9 @@ PARAM( /* func */ NULL ) +#undef H2_SETTING_DESCR +#undef H2_SETTING_NAME + #if 0 /* actual location mgt_param_tbl.c */ PARAM( From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:11 +0000 (UTC) Subject: [6.0] cf444b4fb http2_session: Advertise http_req_size to clients Message-ID: <20240404143311.CCC80103A32@lists.varnish-cache.org> commit cf444b4fbe3d3ba85223a2a4d8a5d2b6d5326c27 Author: Dridi Boukelmoune Date: Thu Mar 28 16:35:52 2024 +0100 http2_session: Advertise http_req_size to clients Since http_req_size was already established for this purpose, and is now enforced for h2 traffic, it should naturally become the basis for the MAX_HEADER_LIST_SIZE setting in the initial SETTINGS frame sent to clients. The h2_max_header_list_size parameter will grow a new purpose. Conflicts: bin/varnishtest/tests/t02000.vtc bin/varnishtest/tests/t02005.vtc include/tbl/params.h diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 06e625f33..2cfba41ef 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -85,6 +85,7 @@ h2_local_settings(struct h2_settings *h2s) h2s->l = cache_param->h2_##l; #include "tbl/h2_settings.h" #undef H2_SETTINGS_PARAM_ONLY + h2s->max_header_list_size = cache_param->http_req_size; } /********************************************************************** diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index 789b5d822..fb3eef341 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -69,7 +69,7 @@ varnish v1 -expect MEMPOOL.sess1.live == 0 process p1 -stop # shell {cat ${tmpdir}/vlog} # SETTINGS with default initial window size -shell -match {1001 H2TxHdr c \[000006040000000000\]} { +shell -match {1001 H2TxHdr c \[00000c040000000000\]} { cat ${tmpdir}/vlog } diff --git a/bin/varnishtest/tests/t02005.vtc b/bin/varnishtest/tests/t02005.vtc index 9aff481c9..a705a8f70 100644 --- a/bin/varnishtest/tests/t02005.vtc +++ b/bin/varnishtest/tests/t02005.vtc @@ -27,7 +27,7 @@ varnish v1 -cliok "param.set debug +syncvsl" logexpect l1 -v v1 -g raw { expect * 1001 ReqAcct "80 7 87 106 8 114" - expect * 1000 ReqAcct "45 8 53 63 28 91" + expect * 1000 ReqAcct "45 8 53 63 34 97" } -start client c1 { diff --git a/include/tbl/h2_settings.h b/include/tbl/h2_settings.h index c382e862d..a81a46d48 100644 --- a/include/tbl/h2_settings.h +++ b/include/tbl/h2_settings.h @@ -90,6 +90,7 @@ H2_SETTING( // rfc7540,l,2150,2157 H2CE_PROTOCOL_ERROR ) +#ifndef H2_SETTINGS_PARAM_ONLY H2_SETTING( // rfc7540,l,2159,2167 MAX_HEADER_LIST_SIZE, max_header_list_size, @@ -99,6 +100,7 @@ H2_SETTING( // rfc7540,l,2159,2167 0xffffffff, 0 ) +#endif #undef H2_SETTING /*lint -restore */ diff --git a/include/tbl/params.h b/include/tbl/params.h index 5ec1ec821..7c3945d33 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -703,11 +703,13 @@ PARAM( /* flags */ 0, /* s-text */ "Maximum number of bytes of HTTP client request we will deal with. " - " This is a limit on all bytes up to the double blank line which " - "ends the HTTP request.\n" + "This is a limit on all bytes up to the double blank line which " + "ends the HTTP request. " "The memory for the request is allocated from the client workspace " "(param: workspace_client) and this parameter limits how much of " - "that the request is allowed to take up.", + "that the request is allowed to take up.\n\n" + "For HTTP2 clients, it is advertised as MAX_HEADER_LIST_SIZE in " + "the initial SETTINGS frame.", /* l-text */ "", /* func */ NULL ) @@ -1899,7 +1901,9 @@ PARAM( /* units */ "bytes", /* flags */ 0, /* s-text */ - "HTTP2 maximum size of an uncompressed header list.", + "HTTP2 maximum size of an uncompressed header list. This parameter " + "is not mapped to " H2_SETTING_NAME(MAX_HEADER_LIST_SIZE) " in the " + "initial SETTINGS frame, the http_req_size parameter is instead.", /* l-text */ "", /* func */ NULL ) From dridi.boukelmoune at gmail.com Thu Apr 4 14:33:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 14:33:11 +0000 (UTC) Subject: [6.0] 62c525e81 http2_hpack: Enforce h2_max_header_list_size Message-ID: <20240404143311.EC920103A3F@lists.varnish-cache.org> commit 62c525e81cb82ee391eeda1a8ec639abdf8af72d Author: Dridi Boukelmoune Date: Thu Mar 28 16:50:39 2024 +0100 http2_hpack: Enforce h2_max_header_list_size This parameter has a new role that consists in interrupting connections when decoding an HPACK block leads to a header list so large that the client must be stopped. By default, too large is 150% of http_req_size. Conflicts: include/tbl/params.h diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 00fbe7c6b..a157e9b29 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -235,6 +235,7 @@ struct h2h_decode { enum vhd_ret_e vhd_ret; char *out; char *reset; + int64_t limit; size_t out_l; size_t out_u; size_t namelen; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 1b476c9aa..e32da14ed 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -276,6 +276,14 @@ h2h_decode_init(const struct h2_sess *h2) XXXAN(d->out_l); d->out = h2->new_req->http->ws->f; d->reset = d->out; + + if (cache_param->h2_max_header_list_size == 0) + d->limit = h2->local_settings.max_header_list_size * 1.5; + else + d->limit = cache_param->h2_max_header_list_size; + + if (d->limit < h2->local_settings.max_header_list_size) + d->limit = INT64_MAX; } /* Possible error returns: @@ -346,7 +354,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) if (d->error != NULL) assert(H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)); - while (1) { + while (d->limit >= 0) { AN(d->out); assert(d->out_u <= d->out_l); d->vhd_ret = VHD_Decode(d->vhd, h2->dectbl, in, in_l, &in_u, @@ -364,6 +372,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) } if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { + d->limit -= d->out_u; d->out_u = 0; assert(d->out_u < d->out_l); continue; @@ -397,6 +406,7 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out[d->out_u++] = '\0'; /* Zero guard */ d->out += d->out_u; d->out_l -= d->out_u; + d->limit -= d->out_u; d->out_u = 0; d->namelen = 0; break; @@ -413,15 +423,25 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { d->out = d->reset; d->out_l = hp->ws->r - d->out; + d->limit -= d->out_u; d->out_u = 0; assert(d->out_u < d->out_l); } else if (d->error) break; } - if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) - return (0); /* Stream error, delay reporting until - h2h_decode_fini so that we can process the - complete header block */ + if (d->limit < 0) { + /* Fatal error, the client exceeded both http_req_size + * and h2_max_header_list_size. */ + VSLb(h2->vsl, SLT_SessError, "Header list too large"); + return (H2CE_ENHANCE_YOUR_CALM); + } + + if (H2_ERROR_MATCH(d->error, H2SE_ENHANCE_YOUR_CALM)) { + /* Stream error, delay reporting until h2h_decode_fini so + * that we can process the complete header block. */ + return (NULL); + } + return (d->error); } diff --git a/bin/varnishtest/tests/f00015.vtc b/bin/varnishtest/tests/f00015.vtc new file mode 100644 index 000000000..de5df8ed2 --- /dev/null +++ b/bin/varnishtest/tests/f00015.vtc @@ -0,0 +1,19 @@ +varnishtest "h2 CONTINUATION flood" + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set vsl_mask -H2RxHdr,-H2RxBody" +varnish v1 -vcl { backend be none; } -start + +client c1 { + non_fatal + stream next { + txreq -nohdrend + loop 1000 { + txcont -nohdrend -hdr noise ${string,repeat,4096,x} + } + txcont -hdr last header + } -run +} -run + +varnish v1 -expect MAIN.s_req_hdrbytes < 65536 +varnish v1 -expect MAIN.sc_overload == 1 diff --git a/include/tbl/params.h b/include/tbl/params.h index 7c3945d33..2cb61260a 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1897,13 +1897,21 @@ PARAM( /* typ */ bytes_u, /* min */ "0b", /* max */ "4294967295b", - /* default */ "4294967295b", + /* default */ "0b", /* units */ "bytes", /* flags */ 0, /* s-text */ "HTTP2 maximum size of an uncompressed header list. This parameter " "is not mapped to " H2_SETTING_NAME(MAX_HEADER_LIST_SIZE) " in the " - "initial SETTINGS frame, the http_req_size parameter is instead.", + "initial SETTINGS frame, the http_req_size parameter is instead." + "The http_req_size advises HTTP2 clients of the maximum size for " + "the header list. Exceeding http_req_size results in a reset stream " + "after processing the HPACK block to perserve the connection, but " + "exceeding h2_max_header_list_size results in the HTTP2 connection " + "going away immediately.\n\n" + "If h2_max_header_list_size is lower than http_req_size, it has no " + "effect, except for the special value zero interpreted as 150% of " + "http_req_size.", /* l-text */ "", /* func */ NULL ) From dridi.boukelmoune at gmail.com Thu Apr 4 18:09:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 18:09:06 +0000 (UTC) Subject: [master] 758dad037 vnum: Add test coverage for 32bit unsigned limits Message-ID: <20240404180906.F16FB1116CC@lists.varnish-cache.org> commit 758dad03772767511e99d340e8d05390c577db6e Author: Dridi Boukelmoune Date: Thu Apr 4 19:15:34 2024 +0200 vnum: Add test coverage for 32bit unsigned limits diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index 021aa7baf..b12e6bc01 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -486,7 +486,7 @@ static struct test_case { uintmax_t rel; uintmax_t val; const char *err; -} test_cases[] = { +} test_vnum_2bytes[] = { { "1", (uintmax_t)0, (uintmax_t)1 }, { "1B", (uintmax_t)0, (uintmax_t)1<<0 }, { "1 B", (uintmax_t)0, (uintmax_t)1<<0 }, @@ -530,6 +530,10 @@ static struct test_case { { "2%", (uintmax_t)1024, (uintmax_t)20 }, { "3%", (uintmax_t)1024, (uintmax_t)30 }, + /* 32bit limits */ + { "4294967295b", (uintmax_t)0, (uintmax_t)4294967295ULL}, + { "4294967294b", (uintmax_t)0, (uintmax_t)4294967294ULL}, + /* Check the error checks */ { "", 0, 0, err_invalid_num }, { "-1", 0, 0, err_invalid_num }, @@ -674,7 +678,7 @@ main(int argc, char *argv[]) } } - for (tc = test_cases; tc->str; ++tc) { + for (tc = test_vnum_2bytes; tc->str; ++tc) { e = VNUM_2bytes(tc->str, &val, tc->rel); if (e != NULL) val = 0; From dridi.boukelmoune at gmail.com Thu Apr 4 18:09:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 18:09:07 +0000 (UTC) Subject: [master] 65a42d183 param: Restore old default h2_max_header_list_size Message-ID: <20240404180907.1605C1116CF@lists.varnish-cache.org> commit 65a42d183eb3da84f5f6ba3150d607092354f450 Author: Dridi Boukelmoune Date: Thu Apr 4 19:18:12 2024 +0200 param: Restore old default h2_max_header_list_size Except that the old default value replaces the maximum one. Aligning with the literal maximum value for the underlying HTTP/2 setting breaks 32bit builds because the byte tweaks take a detour via ssize_t. When it casts to uintmax_t the MSB is propagated all the way, triggering the following error at build time: > 4294967295b is too large for this architecture. Instead of fighting a tweak that is clearly wrong, grant h2 clients a maximum of 2GB of uncompressed headers (instead of 4GB) that will never happen, because h2 is overall much wronger. diff --git a/include/tbl/params.h b/include/tbl/params.h index 1b50d4cda..938b00f4b 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1288,7 +1288,7 @@ PARAM_SIMPLE( /* name */ h2_max_header_list_size, /* type */ bytes_u, /* min */ "0b", - /* max */ "4294967295b", + /* max */ "2147483647b", /* NB: not the RFC maximum */ /* def */ "0b", /* units */ "bytes", /* descr */ From dridi.boukelmoune at gmail.com Thu Apr 4 18:09:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 18:09:07 +0000 (UTC) Subject: [master] 6c27aae63 vtc: Coverage for initial h2 settings Message-ID: <20240404180907.2EF731116D3@lists.varnish-cache.org> commit 6c27aae638799d2d61c33b2672ff7a6407098044 Author: Dridi Boukelmoune Date: Thu Apr 4 19:59:25 2024 +0200 vtc: Coverage for initial h2 settings There's no way to probe the current push status or maximum frame size. diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index 97120a9e7..3f16c6e6c 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -35,7 +35,7 @@ client c1 { varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.reset h2_initial_window_size" -client c1 { +client c2 { stream 1 { txprio -weight 10 -stream 0 } -run @@ -86,7 +86,7 @@ varnish v1 -syntax 4.1 -vcl+backend { } } -client c1 { +client c3 { stream 7 { txreq -url "/uncached" rxresp @@ -105,3 +105,35 @@ client c1 { expect resp.http.C-Sess-XID == resp.http.B-Sess-XID } -run } -run + +# Check default settings + +varnish v1 -cliok "param.reset h2_header_table_size" +varnish v1 -cliok "param.reset h2_max_concurrent_streams" +varnish v1 -cliok "param.reset h2_initial_window_size" +varnish v1 -cliok "param.reset h2_max_frame_size" +varnish v1 -cliok "param.reset h2_max_header_list_size" +varnish v1 -cliok "param.reset http_req_size" + +client c4 { + txpri + stream 0 { + # check initial settings from varnishd + txsettings + rxsettings + expect settings.ack == false + expect settings.push == + expect settings.hdrtbl == + expect settings.maxstreams == 100 + expect settings.winsize == + expect settings.framesize == + expect settings.hdrsize ~ ^(12288|32768)$ + + # check (some) values not set by varnishd + expect stream.window == 65535 + + txsettings -ack + rxsettings + expect settings.ack == true + } -run +} -run From dridi.boukelmoune at gmail.com Thu Apr 4 18:09:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 18:09:07 +0000 (UTC) Subject: [master] 9e0f47f8c vtc(7): Misplaced double-quote Message-ID: <20240404180907.51CDC1116D7@lists.varnish-cache.org> commit 9e0f47f8c45fc9a559f1a5118877cddfc7aee7be Author: Dridi Boukelmoune Date: Thu Apr 4 20:00:38 2024 +0200 vtc(7): Misplaced double-quote diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 1d401ed17..d54e86e10 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1009,7 +1009,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) /* SECTION: stream.spec.zexpect.settings SETTINGS specific * * settings.ack - * "true" if the ACK flag was set, else ""false. + * "true" if the ACK flag was set, else "false". * * settings.push * "true" if the push settings was set to yes, "false" if set to From dridi.boukelmoune at gmail.com Thu Apr 4 19:17:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 19:17:05 +0000 (UTC) Subject: [6.0] fa84da36f vnum: Add test coverage for 32bit unsigned limits Message-ID: <20240404191706.01CAE113ED9@lists.varnish-cache.org> commit fa84da36f535f9a1d7c9a8328e8482ce0725da94 Author: Dridi Boukelmoune Date: Thu Apr 4 19:15:34 2024 +0200 vnum: Add test coverage for 32bit unsigned limits diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index 59e804ec8..47effe4cc 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -238,7 +238,7 @@ static struct test_case { uintmax_t rel; uintmax_t val; const char *err; -} test_cases[] = { +} test_vnum_2bytes[] = { { "1", (uintmax_t)0, (uintmax_t)1 }, { "1B", (uintmax_t)0, (uintmax_t)1<<0 }, { "1 B", (uintmax_t)0, (uintmax_t)1<<0 }, @@ -278,6 +278,10 @@ static struct test_case { { "2%", (uintmax_t)1024, (uintmax_t)20 }, { "3%", (uintmax_t)1024, (uintmax_t)31 }, + /* 32bit limits */ + { "4294967295b", (uintmax_t)0, (uintmax_t)4294967295ULL}, + { "4294967294b", (uintmax_t)0, (uintmax_t)4294967294ULL}, + /* Check the error checks */ { "", 0, 0, err_miss_num }, { "m", 0, 0, err_invalid_num }, @@ -347,7 +351,7 @@ main(int argc, char *argv[]) } } - for (tc = test_cases; tc->str; ++tc) { + for (tc = test_vnum_2bytes; tc->str; ++tc) { e = VNUM_2bytes(tc->str, &val, tc->rel); if (e != NULL) val = 0; From dridi.boukelmoune at gmail.com Thu Apr 4 19:17:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 19:17:06 +0000 (UTC) Subject: [6.0] fd9bf7e62 param: Restore old default h2_max_header_list_size Message-ID: <20240404191706.1B6AC113EDC@lists.varnish-cache.org> commit fd9bf7e62ade376fd580b8a0924fc7f3d2b3138d Author: Dridi Boukelmoune Date: Thu Apr 4 19:18:12 2024 +0200 param: Restore old default h2_max_header_list_size Except that the old default value replaces the maximum one. Aligning with the literal maximum value for the underlying HTTP/2 setting breaks 32bit builds because the byte tweaks take a detour via ssize_t. When it casts to uintmax_t the MSB is propagated all the way, triggering the following error at build time: > 4294967295b is too large for this architecture. Instead of fighting a tweak that is clearly wrong, grant h2 clients a maximum of 2GB of uncompressed headers (instead of 4GB) that will never happen, because h2 is overall much wronger. Conflicts: include/tbl/params.h diff --git a/include/tbl/params.h b/include/tbl/params.h index 2cb61260a..d3732a0d1 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1896,7 +1896,7 @@ PARAM( /* name */ h2_max_header_list_size, /* typ */ bytes_u, /* min */ "0b", - /* max */ "4294967295b", + /* max */ "2147483647b", /* NB: not the RFC maximum */ /* default */ "0b", /* units */ "bytes", /* flags */ 0, From dridi.boukelmoune at gmail.com Thu Apr 4 19:17:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 19:17:06 +0000 (UTC) Subject: [6.0] 4e7fb1a9f vtc: Coverage for initial h2 settings Message-ID: <20240404191706.32524113EDF@lists.varnish-cache.org> commit 4e7fb1a9ff60c6077df82f4d71aa595644bbaa61 Author: Dridi Boukelmoune Date: Thu Apr 4 19:59:25 2024 +0200 vtc: Coverage for initial h2 settings There's no way to probe the current push status or maximum frame size. diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index fb3eef341..3fd903a74 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -35,7 +35,7 @@ client c1 { varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.reset h2_initial_window_size" -client c1 { +client c2 { stream 1 { txprio -weight 10 -stream 0 } -run @@ -85,7 +85,7 @@ varnish v1 -syntax 4.1 -vcl+backend { } } -client c1 { +client c3 { stream 7 { txreq -url "/uncached" rxresp @@ -103,3 +103,35 @@ client c1 { expect resp.http.C-Sess-XID == resp.http.B-Sess-XID } -run } -run + +# Check default settings + +varnish v1 -cliok "param.reset h2_header_table_size" +varnish v1 -cliok "param.reset h2_max_concurrent_streams" +varnish v1 -cliok "param.reset h2_initial_window_size" +varnish v1 -cliok "param.reset h2_max_frame_size" +varnish v1 -cliok "param.reset h2_max_header_list_size" +varnish v1 -cliok "param.reset http_req_size" + +client c4 { + txpri + stream 0 { + # check initial settings from varnishd + txsettings + rxsettings + expect settings.ack == false + expect settings.push == + expect settings.hdrtbl == + expect settings.maxstreams == 100 + expect settings.winsize == + expect settings.framesize == + expect settings.hdrsize ~ ^(12288|32768)$ + + # check (some) values not set by varnishd + expect stream.window == 65535 + + txsettings -ack + rxsettings + expect settings.ack == true + } -run +} -run From dridi.boukelmoune at gmail.com Thu Apr 4 19:17:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 4 Apr 2024 19:17:06 +0000 (UTC) Subject: [6.0] 5c4fbcc16 vtc(7): Misplaced double-quote Message-ID: <20240404191706.59029113EE4@lists.varnish-cache.org> commit 5c4fbcc166e4eb2c70bce193b1f838ff07f26148 Author: Dridi Boukelmoune Date: Thu Apr 4 20:00:38 2024 +0200 vtc(7): Misplaced double-quote diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 7c6a4e563..5d05977f0 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -971,7 +971,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) /* SECTION: stream.spec.zexpect.settings SETTINGS specific * * settings.ack - * "true" if the ACK flag was set, else ""false. + * "true" if the ACK flag was set, else "false". * * settings.push * "true" if the push settings was set to yes, "false" if set to From dridi.boukelmoune at gmail.com Mon Apr 8 06:05:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 8 Apr 2024 06:05:09 +0000 (UTC) Subject: [6.0] a8dd3c8ca Cast to (u)intmax_t for %ju and %jd Message-ID: <20240408060509.B3BA8116D4C@lists.varnish-cache.org> commit a8dd3c8cad8bea29f378a68823f96e36a9cbfe17 Author: Martin Blix Grydeland Date: Wed Sep 22 10:35:27 2021 +0200 Cast to (u)intmax_t for %ju and %jd This to silence errors on OSX where apparently int64_t isn't type equivalent to intmax_t, causing printf-errors when using %jd. Fixes: #3699 diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c index d41eb3b49..0232c72a3 100644 --- a/bin/varnishd/http2/cache_http2_panic.c +++ b/bin/varnishd/http2/cache_http2_panic.c @@ -29,6 +29,8 @@ #include "config.h" +#include + #include "cache/cache_varnishd.h" #include "cache/cache_transport.h" @@ -107,7 +109,7 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp) VSB_printf(vsb, "t_send = %f, t_winupd = %f,\n", r2->t_send, r2->t_winupd); VSB_printf(vsb, "t_window = %jd, r_window = %jd,\n", - r2->t_window, r2->r_window); + (intmax_t)r2->t_window, (intmax_t)r2->r_window); VSB_printf(vsb, "rxbuf = %p", r2->rxbuf); if (r2->rxbuf != NULL) { @@ -117,7 +119,8 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp) VSB_printf(vsb, "stvbuf = %p,\n", r2->rxbuf->stvbuf); VSB_printf(vsb, "{size, tail, head} = {%u, %ju, %ju},\n", - r2->rxbuf->size, r2->rxbuf->tail, r2->rxbuf->head); + r2->rxbuf->size, (uintmax_t)r2->rxbuf->tail, + (uintmax_t)r2->rxbuf->head); VSB_indent(vsb, -2); VSB_printf(vsb, "}"); } From dridi.boukelmoune at gmail.com Mon Apr 8 13:48:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 8 Apr 2024 13:48:05 +0000 (UTC) Subject: [master] 2f4a9c8d5 mgt: Always recreate secret file on startup Message-ID: <20240408134805.A637E124FDC@lists.varnish-cache.org> commit 2f4a9c8d504ffe0bfb85ea242ea48d695c7da2f3 Author: Stephane Cance Date: Fri Apr 5 10:42:14 2024 +0200 mgt: Always recreate secret file on startup As both the varnish working directory and the secret file may pre-exist, this ensures permissions remain restrictive on it. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index c1bdaf812..dc2bce56b 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -272,10 +272,16 @@ make_secret(const char *dirname) assert(asprintf(&fn, "%s/_.secret", dirname) > 0); VJ_master(JAIL_MASTER_FILE); - fdo = open(fn, O_RDWR|O_CREAT|O_TRUNC, 0640); - if (fdo < 0) + if (unlink(fn) < 0 && errno != ENOENT) { + ARGV_ERR("Cannot remove pre-existing secret-file in %s (%s)\n", + dirname, VAS_errtxt(errno)); + } + + fdo = open(fn, O_RDWR|O_CREAT|O_EXCL, 0640); + if (fdo < 0) { ARGV_ERR("Cannot create secret-file in %s (%s)\n", dirname, VAS_errtxt(errno)); + } for (i = 0; i < 256; i++) { AZ(VRND_RandomCrypto(&b, 1)); diff --git a/bin/varnishtest/tests/b00084.vtc b/bin/varnishtest/tests/b00084.vtc new file mode 100644 index 000000000..6797d28e6 --- /dev/null +++ b/bin/varnishtest/tests/b00084.vtc @@ -0,0 +1,42 @@ +varnishtest "make sure an already setup secret file remains protected" + +varnish v1 -vcl { backend default none; } -start + +shell -match _.secret { + find "${tmpdir}"/v1/_.secret -perm 0640 -size 256c +} + +varnish v1 -stop -wait + +shell { + test ! -f "${tmpdir}"/v1/_.secret +} + +# since varnishtest destroys workdir silently before startup +# this must fool varnishtest to not manage the workdir +shell -match _.secret { + set -e + mkdir -p "${tmpdir}"/v2/ + touch "${tmpdir}"/v2/_.secret + chmod 0666 "${tmpdir}"/v2/_.secret + find "${tmpdir}"/v2/_.secret -perm 0666 -size 0c +} + +process p1 "exec varnishd -n ${tmpdir}/v2 -F -f '' -a :0" -start + +# wait for startup and check permissions have changed +shell -match _.secret { + set -e + t=50 + while [ "$t" -gt 0 ] && [ ! -d "${tmpdir}"/v2/_.vsm_mgt ]; do + sleep 0.1 + t=$(($t - 1)) + done + find "${tmpdir}"/v2/_.secret -perm 0640 -size 256c +} + +process p1 -stop -wait + +shell { + test ! -f "${tmpdir}"/v2/_.secret +} diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index 8ce91b74b..090f26147 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -44,7 +44,7 @@ shell -err -expect {Cannot open -S file} { varnishd -S ${tmpdir}/nonexistent -n ${tmpdir}/v0 -f '' } -shell -err -expect {Cannot create secret-file in} { +shell -err -expect {Cannot remove pre-existing secret-file in} { mkdir ${tmpdir}/is_a_dir ${tmpdir}/is_a_dir/_.secret varnishd -n ${tmpdir}/is_a_dir -d -a :0 } From phk at FreeBSD.org Tue Apr 9 16:39:11 2024 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 9 Apr 2024 16:39:11 +0000 (UTC) Subject: [master] 11aa2c982 flexelinting Message-ID: <20240409163911.D95171094E8@lists.varnish-cache.org> commit 11aa2c982807e9ffc312839700c1b9bc6e671bf2 Author: Poul-Henning Kamp Date: Tue Apr 9 16:37:42 2024 +0000 flexelinting diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index c4274a656..5f6f34cf7 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -40,7 +40,7 @@ #include "vct.h" static void -h2h_assert_ready(struct h2h_decode *d) +h2h_assert_ready(const struct h2h_decode *d) { CHECK_OBJ_NOTNULL(d, H2H_DECODE_MAGIC); @@ -280,7 +280,8 @@ h2h_decode_init(const struct h2_sess *h2) d->reset = d->out; if (cache_param->h2_max_header_list_size == 0) - d->limit = h2->local_settings.max_header_list_size * 1.5; + d->limit = + (long)(h2->local_settings.max_header_list_size * 1.5); else d->limit = cache_param->h2_max_header_list_size; diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c index 06ea40c15..227097e6b 100644 --- a/bin/varnishd/http2/cache_http2_panic.c +++ b/bin/varnishd/http2/cache_http2_panic.c @@ -79,7 +79,7 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp) VSB_printf(vsb, "refcnt = %d, bogosity = %d, error = %s\n", h2->refcnt, h2->bogosity, h2_panic_error(h2->error)); VSB_printf(vsb, - "open_streams = %u, highest_stream = %u," + "open_streams = %d, highest_stream = %u," " goaway_last_stream = %u,\n", h2->open_streams, h2->highest_stream, h2->goaway_last_stream); VSB_cat(vsb, "local_settings = {"); diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index d9210ff3c..5d0321160 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -732,7 +732,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) * increase under our feet. */ if (h2->open_streams >= - h2->local_settings.max_concurrent_streams) { + (int)h2->local_settings.max_concurrent_streams) { VSLb(h2->vsl, SLT_Debug, "H2: stream %u: Hit maximum number of " "concurrent streams", h2->rxf_stream); diff --git a/include/vdef.h b/include/vdef.h index a321407d7..b287e595f 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -263,7 +263,7 @@ typedef struct { #define Tcheck(t) do { (void)pdiff((t).b, (t).e); } while (0) #define Tlen(t) (pdiff((t).b, (t).e)) -#define Tstr(s) ((txt){(s), (s) + strlen(s)}) +#define Tstr(s) (/*lint -e(446)*/ (txt){(s), (s) + strlen(s)}) #define Tstrcmp(t, s) (strncmp((t).b, (s), Tlen(t))) /* #3020 dummy definitions until PR is merged*/ From dridi.boukelmoune at gmail.com Tue Apr 9 17:28:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 9 Apr 2024 17:28:07 +0000 (UTC) Subject: [6.0] ead236cf1 flexelinting Message-ID: <20240409172807.1B6AA10DE28@lists.varnish-cache.org> commit ead236cf1cec89338cd6975a17749a020b68eea3 Author: Poul-Henning Kamp Date: Tue Apr 9 16:37:42 2024 +0000 flexelinting diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index e32da14ed..8ffeb8ab2 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -38,7 +38,7 @@ #include "vct.h" static void -h2h_assert_ready(struct h2h_decode *d) +h2h_assert_ready(const struct h2h_decode *d) { CHECK_OBJ_NOTNULL(d, H2H_DECODE_MAGIC); @@ -278,7 +278,8 @@ h2h_decode_init(const struct h2_sess *h2) d->reset = d->out; if (cache_param->h2_max_header_list_size == 0) - d->limit = h2->local_settings.max_header_list_size * 1.5; + d->limit = + (long)(h2->local_settings.max_header_list_size * 1.5); else d->limit = cache_param->h2_max_header_list_size; diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index f7c4c5973..e736d7a36 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -725,7 +725,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) * increase under our feet. */ if (h2->open_streams >= - h2->local_settings.max_concurrent_streams) { + (int)h2->local_settings.max_concurrent_streams) { VSLb(h2->vsl, SLT_Debug, "H2: stream %u: Hit maximum number of " "concurrent streams", h2->rxf_stream); diff --git a/include/vdef.h b/include/vdef.h index 86fdb0a21..8db629af1 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -241,5 +241,5 @@ typedef struct { #define Tcheck(t) do { (void)pdiff((t).b, (t).e); } while (0) #define Tlen(t) (pdiff((t).b, (t).e)) -#define Tstr(s) ((txt){(s), (s) + strlen(s)}) +#define Tstr(s) (/*lint -e(446)*/ (txt){(s), (s) + strlen(s)}) #define Tstrcmp(t, s) (strncmp((t).b, (s), Tlen(t))) From dridi.boukelmoune at gmail.com Mon Apr 22 13:14:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 22 Apr 2024 13:14:12 +0000 (UTC) Subject: [master] 08d4e8045 hash: Close an expiry loophole Message-ID: <20240422131412.439801115C1@lists.varnish-cache.org> commit 08d4e8045502c5c32796c62c6bb2574853a87e12 Author: Dridi Boukelmoune Date: Tue Apr 9 20:09:30 2024 +0200 hash: Close an expiry loophole The opposite of 'EXP_Ttl(req, oc) > req->t_req' should not have been 'EXP_Ttl(NULL, oc) < req->t_req'. If we somehow enter the lookup when the two operands are equal, the objcore suffers a phenomenon known as Schr?dinger's expiry. The chances of running into this scenario range from epsilon to 100%. Because 't_req' is stable across restarts, a soft purge will reliably trigger this case. Test case by Alve Elde who first demonstrated the problem. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 32f9340c7..54bea6192 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -485,7 +485,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) break; } - if (EXP_Ttl(NULL, oc) < req->t_req && /* ignore req.ttl */ + if (EXP_Ttl(NULL, oc) <= req->t_req && /* ignore req.ttl */ oc->t_origin > exp_t_origin) { /* record the newest object */ exp_oc = oc; diff --git a/bin/varnishtest/tests/c00132.vtc b/bin/varnishtest/tests/c00132.vtc new file mode 100644 index 000000000..82df14512 --- /dev/null +++ b/bin/varnishtest/tests/c00132.vtc @@ -0,0 +1,39 @@ +varnishtest "304 revalidations with purge.soft()" + +server s1 { + rxreq + txresp -hdr "etag: foo" -body "foo" + + rxreq + expect req.http.If-None-Match == "foo" + txresp -status 304 -hdr "etag: foo" +} -start + +varnish v1 -vcl+backend { + import purge; + + sub vcl_hit { + if (req.restarts == 0) { + purge.soft(); + return (restart); + } + } + + sub vcl_backend_response { + set beresp.ttl = 1d; + set beresp.grace = 1d; + set beresp.keep = 1d; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.body == "foo" + + txreq + rxresp + expect resp.status == 200 + expect resp.body == "foo" +} -run From nils.goroll at uplex.de Mon Apr 29 13:39:04 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Apr 2024 13:39:04 +0000 (UTC) Subject: [master] 00db3feea vtest: Document expect_pattern Message-ID: <20240429133904.B112111FF0C@lists.varnish-cache.org> commit 00db3feeae860be6794a788c76ae2943d4784279 Author: Nils Goroll Date: Mon Mar 25 11:20:41 2024 +0100 vtest: Document expect_pattern diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 540e8fc1d..5652e1e02 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -373,6 +373,13 @@ cmd_http_expect(CMD_ARGS) vtc_expect(vl, av[0], lhs, cmp, av[2], rhs); } +/* SECTION: client-server.spec.expect_pattern + * + * expect_pattern + * + * Expect as the http body the test pattern generated by chunkedlen ('0'..'7' + * repeating). + */ static void cmd_http_expect_pattern(CMD_ARGS) { From dridi.boukelmoune at gmail.com Mon Apr 29 20:49:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 29 Apr 2024 20:49:11 +0000 (UTC) Subject: [6.0] bec48849f h2: Ensure highest_stream is consistent also on Upgrade: h2c Message-ID: <20240429204911.ADB4C103CC1@lists.varnish-cache.org> commit bec48849fba4146aa707d2f3d59089ba24944f21 Author: Dag Haavi Finstad Date: Thu Apr 25 14:36:57 2024 +0200 h2: Ensure highest_stream is consistent also on Upgrade: h2c Conflicts: bin/varnishd/http2/cache_http2_session.c diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 2cfba41ef..c6a742ffa 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -289,6 +289,8 @@ h2_ou_session(struct worker *wrk, struct h2_sess *h2, /* Start req thread */ r2 = h2_new_req(wrk, h2, 1, req); + AZ(h2->highest_stream); + h2->highest_stream = r2->stream; req->transport = &H2_transport; req->req_step = R_STP_TRANSPORT; req->task.func = h2_do_req; diff --git a/bin/varnishtest/tests/t02001.vtc b/bin/varnishtest/tests/t02001.vtc index 1da6fb3b5..0e02e82be 100644 --- a/bin/varnishtest/tests/t02001.vtc +++ b/bin/varnishtest/tests/t02001.vtc @@ -1,6 +1,6 @@ varnishtest "H1->H2 Upgrade" -barrier b1 cond 2 +barrier b1 cond 2 -cyclic server s1 { rxreq @@ -37,6 +37,11 @@ server s1 { expect req.http.host == foo.bar barrier b1 sync txresp -status 402 -bodylen 11 + + rxreq + expect req.url == /upgrade3 + barrier b1 sync + txresp -status 200 } -start varnish v1 -vsl_catchup @@ -156,3 +161,37 @@ varnish v1 -expect MEMPOOL.req0.live == 0 varnish v1 -expect MEMPOOL.req1.live == 0 varnish v1 -expect MEMPOOL.sess0.live == 0 varnish v1 -expect MEMPOOL.sess1.live == 0 + +# Upgrade: h2c followed by activity on the newly opened stream +client c1 { + send "GET /upgrade3 HTTP/1.1\r\n" + send "Host: foo.bar\r\n" + send "Upgrade: h2c\r\n" + send "HTTP2-Settings: AAMAAABkAAQAAP__\r\n" + send "\r\n" + rxresp + expect resp.status == 101 + expect resp.http.upgrade == h2c + expect resp.http.connection == Upgrade + txpri + stream 0 { + rxsettings + txsettings + txsettings -ack + rxsettings + expect settings.ack == true + } -run + barrier b1 sync + stream 1 { + txwinup -size 256 + rxresp + expect resp.status == 200 + } -run +} -run + +varnish v1 -vsl_catchup + +varnish v1 -expect MEMPOOL.req0.live == 0 +varnish v1 -expect MEMPOOL.req1.live == 0 +varnish v1 -expect MEMPOOL.sess0.live == 0 +varnish v1 -expect MEMPOOL.sess1.live == 0 From phk at FreeBSD.org Tue Apr 30 20:22:07 2024 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 30 Apr 2024 20:22:07 +0000 (UTC) Subject: [master] 8643ead83 Simplify VFIL_allocate() by using posix_fallocate(2) when insisting (=VSHM) Message-ID: <20240430202207.4B36110EFFE@lists.varnish-cache.org> commit 8643ead83fe32b0027992e1254dac3f09c059f6a Author: Poul-Henning Kamp Date: Tue Apr 30 20:20:54 2024 +0000 Simplify VFIL_allocate() by using posix_fallocate(2) when insisting (=VSHM) diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c index 8d1f3ab60..4df3999d0 100644 --- a/lib/libvarnish/vfil.c +++ b/lib/libvarnish/vfil.c @@ -230,11 +230,6 @@ VFIL_allocate(int fd, uintmax_t size, int insist) { struct stat st; uintmax_t fsspace; - size_t l; - ssize_t l2, l3; - char *buf; - ssize_t bufsiz; - int retval = 0; if (ftruncate(fd, size)) return (-1); @@ -248,6 +243,10 @@ VFIL_allocate(int fd, uintmax_t size, int insist) errno = ENOSPC; return (-1); } + if (insist) { + /* This falls back to writing zeros, as we want */ + return (posix_fallocate(fd, 0, size) ? -1 : 0); + } #if defined(__linux__) && defined(HAVE_FALLOCATE) { /* fallocate will for some filesystems (e.g. xfs) not take @@ -267,31 +266,7 @@ VFIL_allocate(int fd, uintmax_t size, int insist) } } #endif - if (!insist) - return (0); - - /* Write size zero bytes to make sure the entire file is allocated - in the file system */ - if (size > 65536) - bufsiz = 64 * 1024; - else - bufsiz = size; - buf = calloc(1, bufsiz); - AN(buf); - assert(lseek(fd, 0, SEEK_SET) == 0); - for (l = 0; l < size; l += l2) { - l2 = bufsiz; - if (l + l2 > size) - l2 = size - l; - l3 = write(fd, buf, l2); - if (l3 != l2) { - retval = -1; - break; - } - } - assert(lseek(fd, 0, SEEK_SET) == 0); - free(buf); - return (retval); + return (0); } struct vfil_dir { From phk at FreeBSD.org Tue Apr 30 22:09:06 2024 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 30 Apr 2024 22:09:06 +0000 (UTC) Subject: [master] cc35e776b Revert previous commit, it breaks sunos Message-ID: <20240430220907.0541F11275C@lists.varnish-cache.org> commit cc35e776b816bb99ff14f6c567842bc1950cf0ad Author: Poul-Henning Kamp Date: Tue Apr 30 20:36:08 2024 +0000 Revert previous commit, it breaks sunos diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c index 4df3999d0..8d1f3ab60 100644 --- a/lib/libvarnish/vfil.c +++ b/lib/libvarnish/vfil.c @@ -230,6 +230,11 @@ VFIL_allocate(int fd, uintmax_t size, int insist) { struct stat st; uintmax_t fsspace; + size_t l; + ssize_t l2, l3; + char *buf; + ssize_t bufsiz; + int retval = 0; if (ftruncate(fd, size)) return (-1); @@ -243,10 +248,6 @@ VFIL_allocate(int fd, uintmax_t size, int insist) errno = ENOSPC; return (-1); } - if (insist) { - /* This falls back to writing zeros, as we want */ - return (posix_fallocate(fd, 0, size) ? -1 : 0); - } #if defined(__linux__) && defined(HAVE_FALLOCATE) { /* fallocate will for some filesystems (e.g. xfs) not take @@ -266,7 +267,31 @@ VFIL_allocate(int fd, uintmax_t size, int insist) } } #endif - return (0); + if (!insist) + return (0); + + /* Write size zero bytes to make sure the entire file is allocated + in the file system */ + if (size > 65536) + bufsiz = 64 * 1024; + else + bufsiz = size; + buf = calloc(1, bufsiz); + AN(buf); + assert(lseek(fd, 0, SEEK_SET) == 0); + for (l = 0; l < size; l += l2) { + l2 = bufsiz; + if (l + l2 > size) + l2 = size - l; + l3 = write(fd, buf, l2); + if (l3 != l2) { + retval = -1; + break; + } + } + assert(lseek(fd, 0, SEEK_SET) == 0); + free(buf); + return (retval); } struct vfil_dir {