From e79079b176fc7f2da53f8b2664f0bad8615ab2bc Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Tue, 4 Aug 2026 13:15:12 -0500 Subject: [PATCH] Harden AuTests against timing races Two AuTests can fail depending on process scheduling and response framing. The HTTP/2 chunked clients can reach their origins before the listeners are ready, while the capped stale-response fetch can stop at a body-block boundary and time out instead of exercising its memory fallback. This patch uses a deterministic raw origin that ignores readiness probes, serves one real request, and exits. It also sizes and documents the stale-response header at the cap-plus-sentinel read boundary so memory-limit rejection is independent of body segmentation. --- .../chunked_encoding_h2.test.py | 21 ++-- .../chunked_encoding_h2_server.py | 115 ++++++++++++++++++ .../chunked_encoding/delay-server.sh | 43 ------- tests/gold_tests/chunked_encoding/server2.sh | 41 ------- tests/gold_tests/chunked_encoding/server3.sh | 41 ------- .../stale_response_max_memory.replay.yaml | 5 + 6 files changed, 132 insertions(+), 134 deletions(-) create mode 100644 tests/gold_tests/chunked_encoding/chunked_encoding_h2_server.py delete mode 100755 tests/gold_tests/chunked_encoding/delay-server.sh delete mode 100755 tests/gold_tests/chunked_encoding/server2.sh delete mode 100755 tests/gold_tests/chunked_encoding/server3.sh diff --git a/tests/gold_tests/chunked_encoding/chunked_encoding_h2.test.py b/tests/gold_tests/chunked_encoding/chunked_encoding_h2.test.py index cbf63581782..ed43095a6ec 100644 --- a/tests/gold_tests/chunked_encoding/chunked_encoding_h2.test.py +++ b/tests/gold_tests/chunked_encoding/chunked_encoding_h2.test.py @@ -16,6 +16,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import sys + Test.Summary = ''' Test interaction of H2 and chunked encoding ''' @@ -36,12 +39,13 @@ # add ssl materials like key, certificates for the server ts.addDefaultSSLFiles() +origin_server = os.path.join(Test.TestDirectory, "chunked_encoding_h2_server.py") delay_server = Test.Processes.Process( - "delay-server", "bash -c '" + Test.TestDirectory + "/delay-server.sh {} outserver1'".format(Test.Variables.upstream_port)) + "delay-server", f'{sys.executable} "{origin_server}" 127.0.0.1 {Test.Variables.upstream_port} outserver1 delayed-chunked') server2 = Test.Processes.Process( - "server2", "bash -c '" + Test.TestDirectory + "/server2.sh {} outserver2'".format(Test.Variables.upstream_port2)) + "server2", f'{sys.executable} "{origin_server}" 127.0.0.1 {Test.Variables.upstream_port2} outserver2 content-length') server3 = Test.Processes.Process( - "server3", "bash -c '" + Test.TestDirectory + "/server3.sh {} outserver3'".format(Test.Variables.upstream_port3)) + "server3", f'{sys.executable} "{origin_server}" 127.0.0.1 {Test.Variables.upstream_port3} outserver3 chunked') ts.Disk.records_config.update( { @@ -64,9 +68,8 @@ ssl_key_name: server.key """.split("\n")) -# Using netcat as a cheap origin server in case 1 so we can insert a delay in sending back the response. -# Replaced microserver for cases 2 and 3 as well because I was getting python exceptions when running -# microserver if chunked encoding headers were specified for the request headers +# Use a raw origin server in case 1 so the final chunk can be delayed. Use it +# for cases 2 and 3 as well because microserver rejects chunked request headers. # H2 GET request # chunked response without content-length @@ -76,7 +79,7 @@ tr.Processes.Default.Command = 'nghttp -vv https://127.0.0.1:{}/delay-chunked-response'.format(ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(Test.Processes.ts) -tr.Processes.Default.StartBefore(delay_server) +tr.Processes.Default.StartBefore(delay_server, ready=When.PortOpen(Test.Variables.upstream_port)) tr.Processes.Default.Streams.All = Testers.ExcludesExpression("RST_STREAM", "Delayed chunk close should not cause reset") tr.Processes.Default.Streams.All += Testers.ExcludesExpression("< content-length", "Should return chunked") tr.Processes.Default.Streams.All += Testers.ContainsExpression(":status: 200", "Should get successful response") @@ -87,7 +90,7 @@ # HTTP2 POST: www.example.com Host, chunked body server2_out = Test.Disk.File("outserver2") tr = Test.AddTestRun() -tr.Processes.Default.StartBefore(server2) +tr.Processes.Default.StartBefore(server2, ready=When.PortOpen(Test.Variables.upstream_port2)) tr.MakeCurlCommand( '--http2 -k https://127.0.0.1:{}/post-full --verbose -H "Transfer-encoding: chunked" -d "Knock knock"'.format( ts.Variables.ssl_port), @@ -103,7 +106,7 @@ # HTTP2 POST: chunked post body and chunked response server3_out = Test.Disk.File("outserver3") tr = Test.AddTestRun() -tr.Processes.Default.StartBefore(server3) +tr.Processes.Default.StartBefore(server3, ready=When.PortOpen(Test.Variables.upstream_port3)) tr.MakeCurlCommand( '--http2 -k https://127.0.0.1:{}/post-chunked --verbose -H "Transfer-encoding: chunked" -d "Knock knock"'.format( ts.Variables.ssl_port), diff --git a/tests/gold_tests/chunked_encoding/chunked_encoding_h2_server.py b/tests/gold_tests/chunked_encoding/chunked_encoding_h2_server.py new file mode 100644 index 00000000000..e1b003c9b0b --- /dev/null +++ b/tests/gold_tests/chunked_encoding/chunked_encoding_h2_server.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Serve one raw HTTP request for the chunked HTTP/2 AuTest.""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from pathlib import Path +import socket +import sys +import time + + +def parse_args() -> argparse.Namespace: + """Parse the command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("address", help="Address on which to listen.") + parser.add_argument("port", type=int, help="Port on which to listen.") + parser.add_argument("output", type=Path, help="File in which to record the request.") + parser.add_argument("response", choices=("delayed-chunked", "content-length", "chunked"), help="Response to send.") + return parser.parse_args() + + +def make_listening_socket(address: str, port: int) -> socket.socket: + """Create and return a listening TCP socket.""" + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((address, port)) + listener.listen(1) + return listener + + +def receive_request(conn: socket.socket) -> bytes: + """Receive one HTTP request, including its declared body.""" + request = b"" + while b"\r\n\r\n" not in request: + data = conn.recv(4096) + if not data: + return request + request += data + + header, _, body = request.partition(b"\r\n\r\n") + content_length = 0 + is_chunked = False + for field in header.split(b"\r\n")[1:]: + name, separator, value = field.partition(b":") + if not separator: + continue + name = name.strip().lower() + value = value.strip().lower() + if name == b"content-length": + content_length = int(value) + elif name == b"transfer-encoding" and b"chunked" in value: + is_chunked = True + + if is_chunked: + while not (body.startswith(b"0\r\n\r\n") or b"\r\n0\r\n\r\n" in body): + data = conn.recv(4096) + if not data: + break + body += data + else: + while len(body) < content_length: + data = conn.recv(4096) + if not data: + break + body += data + + return header + b"\r\n\r\n" + body + + +def send_response(conn: socket.socket, response: str) -> None: + """Send the selected raw HTTP response.""" + if response == "delayed-chunked": + conn.sendall(b"HTTP/1.1 200\r\nTransfer-encoding: chunked\r\n\r\n") + conn.sendall(b"F\r\n123456789012345\r\n") + time.sleep(1) + conn.sendall(b"0\r\n\r\n") + elif response == "content-length": + conn.sendall(b"HTTP/1.1 200\r\nContent-length: 15\r\n\r\n123456789012345") + else: + conn.sendall(b"HTTP/1.1 200\r\nTransfer-encoding: chunked\r\n\r\nF\r\n123456789012345\r\n0\r\n\r\n") + + +def main() -> int: + """Ignore readiness probes, serve one request, and exit.""" + args = parse_args() + with make_listening_socket(args.address, args.port) as listener: + while True: + conn, _ = listener.accept() + with conn: + request = receive_request(conn) + if not request: + # When.PortOpen probes the listener without sending data. + continue + args.output.write_bytes(request) + send_response(conn, args.response) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/gold_tests/chunked_encoding/delay-server.sh b/tests/gold_tests/chunked_encoding/delay-server.sh deleted file mode 100755 index c4d4846ddc3..00000000000 --- a/tests/gold_tests/chunked_encoding/delay-server.sh +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A very simple cleartext server for one HTTP transaction. Does no validation of the Request message. -# Sends a fixed response message - -response () -{ - # Wait for end of Request message. - # - while (( 1 == 1 )) - do - if [[ -f $outfile ]] ; then - if tr '\r\n' '=!' < $outfile | grep '=!=!' > /dev/null - then - break; - fi - fi - sleep 1 - done - - # delay before finishing the chunk - printf "HTTP/1.1 200\r\nTransfer-encoding: chunked\r\n\r\n" - printf "F\r\n123456789012345\r\n" - sleep 1 - printf "0\r\n\r\n" - -} -outfile=$2 -response | nc -l $1 > "$outfile" diff --git a/tests/gold_tests/chunked_encoding/server2.sh b/tests/gold_tests/chunked_encoding/server2.sh deleted file mode 100755 index 2fd88f60da4..00000000000 --- a/tests/gold_tests/chunked_encoding/server2.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A very simple cleartext server for one HTTP transaction. Does no validation of the Request message. -# Sends a fixed response message - - -response () -{ - # Wait for end of Request message. - # - while (( 1 == 1 )) - do - if [[ -f $outfile ]] ; then - if tr '\r\n' '=!' < $outfile | grep '=!=!' > /dev/null - then - break; - fi - fi - sleep 1 - done - - printf "HTTP/1.1 200\r\nContent-length: 15\r\n\r\n" - printf "123456789012345" - -} -outfile=$2 -response | nc -l $1 > "$outfile" diff --git a/tests/gold_tests/chunked_encoding/server3.sh b/tests/gold_tests/chunked_encoding/server3.sh deleted file mode 100755 index e08087faefa..00000000000 --- a/tests/gold_tests/chunked_encoding/server3.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A very simple cleartext server for one HTTP transaction. Does no validation of the Request message. -# Sends a fixed response message - - -response () -{ - # Wait for end of Request message. - # - while (( 1 == 1 )) - do - if [[ -f $outfile ]] ; then - if tr '\r\n' '=!' < $outfile | grep '=!=!' > /dev/null - then - break; - fi - fi - sleep 1 - done - - printf "HTTP/1.1 200\r\nTransfer-encoding: chunked\r\n\r\n" - printf "F\r\n123456789012345\r\n0\r\n\r\n" - -} -outfile=$2 -response | nc -l $1 > "$outfile" diff --git a/tests/gold_tests/pluginTest/stale_response/stale_response_max_memory.replay.yaml b/tests/gold_tests/pluginTest/stale_response/stale_response_max_memory.replay.yaml index 3b199dd6285..af031c9e7b7 100644 --- a/tests/gold_tests/pluginTest/stale_response/stale_response_max_memory.replay.yaml +++ b/tests/gold_tests/pluginTest/stale_response/stale_response_max_memory.replay.yaml @@ -77,6 +77,11 @@ sessions: - [ Connection, close ] - [ Cache-Control, "max-age=1" ] - [ X-Response, oversized-origin-response ] + # The preceding fields serialize to 215 bytes. X-Padding adds 42 wire + # bytes (name, separator, 29-byte value, and CRLF), bringing the header + # to the 256-byte limit plus the one-byte overflow sentinel. This makes + # the memory rejection independent of body segmentation. + - [ X-Padding, aaaaaaaaaaaaaaaaaaaaaaaaaaaaa ] content: size: 512