mirror of
https://github.com/Flowseal/tg-ws-proxy.git
synced 2026-10-21 02:58:45 +00:00
multiplexing media through cfproxy
This commit is contained in:
Flowseal committed
1 parent
fe1580cd83
commit
48954ff9ab
26 files changed
+3598
-23
No files matched your search
@@ -0,0 +1,912 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
import struct
|
||||
import time
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
|
||||
from proxy.balancer import _Balancer
|
||||
from proxy.cf_h2 import (
|
||||
CfH2Pool, MAX_CHANNEL_REQUESTS, MAX_PACKET, _HttpChannel, _HttpLane,
|
||||
_ReplayPacket, _encode_reply, _read_packet, bridge_h2,
|
||||
)
|
||||
from proxy.utils import PROTO_TAG_ABRIDGED, PROTO_TAG_INTERMEDIATE, PROTO_TAG_SECURE
|
||||
|
||||
class _IdentityCipher:
|
||||
def update(self, value):
|
||||
return value
|
||||
|
||||
|
||||
class NativeFramingTest(unittest.IsolatedAsyncioTestCase):
|
||||
async def _packet(self, wire, tag):
|
||||
reader = asyncio.StreamReader()
|
||||
async def feed():
|
||||
for byte in wire:
|
||||
reader.feed_data(bytes([byte]))
|
||||
await asyncio.sleep(0)
|
||||
reader.feed_eof()
|
||||
task = asyncio.create_task(feed())
|
||||
try:
|
||||
return await _read_packet(reader, _IdentityCipher(), tag)
|
||||
finally:
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
async def test_fragmented_abridged_extended_and_quick_ack(self):
|
||||
body = b'a' * 512
|
||||
decoded, quick = await self._packet(b'\xff\x80\x00\x00' + body, PROTO_TAG_ABRIDGED)
|
||||
self.assertEqual(decoded, body)
|
||||
self.assertTrue(quick)
|
||||
|
||||
async def test_intermediate_quick_ack_does_not_change_body(self):
|
||||
body = b'a' * 40
|
||||
decoded, quick = await self._packet(struct.pack('<I', len(body) | 0x80000000) + body,
|
||||
PROTO_TAG_INTERMEDIATE)
|
||||
self.assertEqual(decoded, body)
|
||||
self.assertTrue(quick)
|
||||
|
||||
async def test_padded_plaintext_removes_all_fifteen_padding_lengths(self):
|
||||
body = b'\x00' * 8 + b'm' * 8 + struct.pack('<I', 20) + b'p' * 20
|
||||
for padding in range(16):
|
||||
wire = body + b'z' * padding
|
||||
decoded, quick = await self._packet(struct.pack('<I', len(wire)) + wire, PROTO_TAG_SECURE)
|
||||
self.assertEqual(decoded, body, padding)
|
||||
self.assertFalse(quick)
|
||||
|
||||
async def test_padded_ciphertext_removes_full_padding_blocks(self):
|
||||
body = b'k' * 8 + b'm' * 16 + b'c' * 32
|
||||
for padding in range(16):
|
||||
wire = body + b'z' * padding
|
||||
decoded, _ = await self._packet(struct.pack('<I', len(wire)) + wire, PROTO_TAG_SECURE)
|
||||
self.assertEqual(decoded, body, padding)
|
||||
|
||||
async def test_oversize_header_rejected_before_body_read(self):
|
||||
reader = asyncio.StreamReader()
|
||||
reader.feed_data(struct.pack('<I', MAX_PACKET + 4))
|
||||
with self.assertRaisesRegex(ValueError, 'length'):
|
||||
await _read_packet(reader, _IdentityCipher(), PROTO_TAG_INTERMEDIATE)
|
||||
|
||||
def test_negative_transport_error_keeps_native_framing(self):
|
||||
error = struct.pack('<i', -404)
|
||||
self.assertEqual(_encode_reply(error, PROTO_TAG_ABRIDGED), b'\x01' + error)
|
||||
self.assertEqual(_encode_reply(error, PROTO_TAG_INTERMEDIATE), b'\x04\x00\x00\x00' + error)
|
||||
|
||||
|
||||
class HttpMultiplexTest(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.lane = _HttpLane('kws4.example.org', 1)
|
||||
await self.lane.client.aclose()
|
||||
self.channels = []
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.lane.close()
|
||||
|
||||
def _channel(self):
|
||||
channel = _HttpChannel(self.lane, len(self.channels) + 1, 'owned-test')
|
||||
self.channels.append(channel)
|
||||
return channel
|
||||
|
||||
def _transport(self, handler, *, trace_send=True):
|
||||
async def handle(request):
|
||||
if trace_send and 'trace' in request.extensions:
|
||||
# MockTransport receives the complete body before calling us.
|
||||
await request.extensions['trace']('http2.send_request_body.complete', {'stream_id': 3})
|
||||
result = handler(request)
|
||||
return await result if inspect.isawaitable(result) else result
|
||||
self.lane.client = httpx.AsyncClient(http2=True, transport=httpx.MockTransport(handle))
|
||||
|
||||
async def test_same_auth_prefix_out_of_order_replies_stay_with_source_channel(self):
|
||||
first_started, release_first = asyncio.Event(), asyncio.Event()
|
||||
body_a, body_b = b'samekey!' + b'a' * 32, b'samekey!' + b'b' * 32
|
||||
async def handler(request):
|
||||
if request.content == body_a:
|
||||
first_started.set()
|
||||
await release_first.wait()
|
||||
return httpx.Response(200, content=request.content, extensions={
|
||||
'http_version': b'HTTP/2', 'stream_id': 3 if request.content == body_a else 5})
|
||||
self._transport(handler)
|
||||
first, second = self._channel(), self._channel()
|
||||
await first.send(body_a, False)
|
||||
await first_started.wait()
|
||||
await second.send(body_b, False)
|
||||
self.assertEqual(await asyncio.wait_for(second.receive(), 1), body_b)
|
||||
release_first.set()
|
||||
self.assertEqual(await asyncio.wait_for(first.receive(), 1), body_a)
|
||||
self.assertEqual(self.lane.max_inflight, 2)
|
||||
|
||||
async def test_close_one_channel_does_not_cancel_another(self):
|
||||
async def handler(request):
|
||||
if request.content[:1] == b'a':
|
||||
await asyncio.Event().wait()
|
||||
return httpx.Response(200, content=request.content, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
first, second = self._channel(), self._channel()
|
||||
await first.send(b'a' * 40, False)
|
||||
await first.close()
|
||||
await second.send(b'b' * 40, False)
|
||||
self.assertEqual(await asyncio.wait_for(second.receive(), 1), b'b' * 40)
|
||||
await second.close()
|
||||
self.assertEqual(self.lane.inflight, 0)
|
||||
self.assertEqual(self.lane.queued_bytes, 0)
|
||||
self.assertFalse(self.lane.closed)
|
||||
|
||||
async def test_backpressure_waits_instead_of_dropping_requests(self):
|
||||
release = asyncio.Event()
|
||||
async def handler(request):
|
||||
await release.wait()
|
||||
return httpx.Response(200, content=request.content, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
for _ in range(MAX_CHANNEL_REQUESTS):
|
||||
await channel.send(b'x' * 40, False)
|
||||
extra = asyncio.create_task(channel.send(b'y' * 40, False))
|
||||
await asyncio.sleep(0.01)
|
||||
self.assertFalse(extra.done())
|
||||
self.assertEqual(len(channel.pending), MAX_CHANNEL_REQUESTS)
|
||||
release.set()
|
||||
await asyncio.wait_for(extra, 1)
|
||||
received = [await asyncio.wait_for(channel.receive(), 1) for _ in range(MAX_CHANNEL_REQUESTS + 1)]
|
||||
self.assertEqual(received.count(b'y' * 40), 1)
|
||||
|
||||
async def test_http_error_closes_only_its_channel_without_replaying_packet(self):
|
||||
calls = []
|
||||
async def handler(request):
|
||||
calls.append(request.content)
|
||||
return httpx.Response(403, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
await channel.send(b'x' * 40, False)
|
||||
with self.assertRaisesRegex(ConnectionError, '403'):
|
||||
await asyncio.wait_for(channel.receive(), 1)
|
||||
self.assertEqual(calls, [b'x' * 40])
|
||||
self.assertTrue(channel.closed)
|
||||
self.assertFalse(self.lane.closed)
|
||||
|
||||
async def test_long_polls_cannot_block_fresh_request_and_retired_packet_recovers(self):
|
||||
bodies = [b'samekey!' + bytes([index]) * 32 for index in range(MAX_CHANNEL_REQUESTS)]
|
||||
fresh = b'samekey!' + b'n' * 32
|
||||
calls = []
|
||||
async def handler(request):
|
||||
calls.append(request.content)
|
||||
await request.extensions['trace']('http2.send_request_body.complete', {'stream_id': len(calls) * 2 + 1})
|
||||
if request.content == fresh:
|
||||
return httpx.Response(200, content=b'n' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
if request.content == bodies[0] and calls.count(bodies[0]) == 2:
|
||||
return httpx.Response(200, content=b'r' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
await asyncio.Event().wait()
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
for body in bodies:
|
||||
await channel.send(body, False)
|
||||
await asyncio.sleep(0)
|
||||
original = next(iter(channel.pending))
|
||||
with patch('proxy.cf_h2.REPLAY_REQUEST_SECONDS', .02), \
|
||||
patch('proxy.cf_h2.REPLAY_IDLE_SECONDS', .01), \
|
||||
patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005):
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
await asyncio.wait_for(channel.send(fresh, False), 1)
|
||||
received = [await asyncio.wait_for(channel.receive(), 1) for _ in range(2)]
|
||||
self.assertCountEqual(received, [b'n' * 40, b'r' * 40])
|
||||
self.assertTrue(original.cancelled())
|
||||
self.assertTrue(channel.replay_history[0].retired)
|
||||
self.assertEqual(calls.count(bodies[0]), 2)
|
||||
self.assertEqual(channel.capacity_rotations, 1)
|
||||
self.assertEqual(self.lane.max_inflight, MAX_CHANNEL_REQUESTS)
|
||||
self.assertEqual(self.lane.errors, 0)
|
||||
self.assertFalse(channel.closed)
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
|
||||
async def test_shared_lane_long_polls_cannot_block_new_channel(self):
|
||||
fresh = b'samekey!' + b'n' * 32
|
||||
async def handler(request):
|
||||
await request.extensions['trace']('http2.send_request_body.complete', {'stream_id': 3})
|
||||
if request.content == fresh:
|
||||
return httpx.Response(200, content=b'n' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
await asyncio.Event().wait()
|
||||
self._transport(handler)
|
||||
first, second, waiting = self._channel(), self._channel(), self._channel()
|
||||
with patch('proxy.cf_h2.MAX_LANE_REQUESTS', 2), \
|
||||
patch('proxy.cf_h2.REPLAY_REQUEST_SECONDS', .02), \
|
||||
patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005):
|
||||
await first.send(b'samekey!' + b'a' * 32, False)
|
||||
await second.send(b'samekey!' + b'b' * 32, False)
|
||||
await asyncio.sleep(0)
|
||||
originals = [next(iter(channel.pending)) for channel in (first, second)]
|
||||
recovery = asyncio.create_task(waiting._recover())
|
||||
try:
|
||||
await asyncio.wait_for(waiting.send(fresh, False), 1)
|
||||
self.assertEqual(await asyncio.wait_for(waiting.receive(), 1), b'n' * 40)
|
||||
self.assertEqual(sum(task.cancelled() for task in originals), 1)
|
||||
self.assertEqual(sum(channel.capacity_rotations for channel in (first, second)), 1)
|
||||
self.assertEqual(self.lane.capacity_rotations, 1)
|
||||
self.assertEqual(self.lane.max_inflight, 2)
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
|
||||
async def test_upload_room_accounts_for_waiting_packet_size(self):
|
||||
for limit in ('MAX_CHANNEL_BYTES', 'MAX_LANE_BYTES'):
|
||||
with self.subTest(limit=limit):
|
||||
started = asyncio.Event()
|
||||
async def handler(request):
|
||||
await request.extensions['trace']('http2.send_request_body.complete', {'stream_id': 3})
|
||||
if request.content == b'k' * 40:
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
return httpx.Response(200, content=b'n' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
with patch('proxy.cf_h2.' + limit, 60), \
|
||||
patch('proxy.cf_h2.REPLAY_REQUEST_SECONDS', .02), \
|
||||
patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005):
|
||||
await channel.send(b'k' * 40, False)
|
||||
await started.wait()
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
# 20 bytes are free, but the pending upload needs 40.
|
||||
await asyncio.wait_for(channel.send(b'n' * 40, False), .5)
|
||||
self.assertEqual(await asyncio.wait_for(channel.receive(), .5), b'n' * 40)
|
||||
self.assertEqual(channel.capacity_rotations, 1)
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
await channel.close()
|
||||
|
||||
async def test_close_releases_request_cancelled_before_coroutine_start_once(self):
|
||||
self._transport(lambda request: self.fail('cancelled request reached HTTP'))
|
||||
channel = self._channel()
|
||||
create_task = asyncio.create_task
|
||||
def cancelled_task(coroutine):
|
||||
task = create_task(coroutine)
|
||||
task.cancel()
|
||||
return task
|
||||
with patch('proxy.cf_h2.asyncio.create_task', cancelled_task):
|
||||
await channel.send(b'k' * 40, False)
|
||||
packet = channel.replay_history[0]
|
||||
await channel.close()
|
||||
await channel.close()
|
||||
self.assertEqual((self.lane.inflight, self.lane.queued_bytes, channel.pending_bytes), (0, 0, 0))
|
||||
self.assertFalse(channel.pending)
|
||||
self.assertFalse(channel.pending_since)
|
||||
self.assertFalse(channel.sent_since)
|
||||
self.assertFalse(packet.originals)
|
||||
|
||||
async def test_capacity_rotation_keeps_upload_and_started_response_intact(self):
|
||||
uploading, receiving, waiting = [b'samekey!' + bytes([index]) * 32 for index in range(3)]
|
||||
fresh = b'samekey!' + b'n' * 32
|
||||
body_started = asyncio.Event()
|
||||
class SlowBody(httpx.AsyncByteStream):
|
||||
async def __aiter__(self):
|
||||
yield b'r' * 20
|
||||
body_started.set()
|
||||
await asyncio.Event().wait()
|
||||
async def handler(request):
|
||||
if request.content == uploading:
|
||||
await asyncio.Event().wait() # No send_request_body.complete yet.
|
||||
await request.extensions['trace']('http2.send_request_body.complete', {'stream_id': 3})
|
||||
if request.content == receiving:
|
||||
return httpx.Response(200, stream=SlowBody(), extensions={'http_version': b'HTTP/2'})
|
||||
if request.content == waiting:
|
||||
await asyncio.Event().wait()
|
||||
return httpx.Response(200, content=b'n' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler, trace_send=False)
|
||||
channel = self._channel()
|
||||
with patch('proxy.cf_h2.MAX_CHANNEL_REQUESTS', 3), \
|
||||
patch('proxy.cf_h2.REPLAY_REQUEST_SECONDS', .02), \
|
||||
patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005):
|
||||
for body in (uploading, receiving, waiting):
|
||||
await channel.send(body, False)
|
||||
await asyncio.wait_for(body_started.wait(), 1)
|
||||
tasks = list(channel.pending)
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
await asyncio.wait_for(channel.send(fresh, False), 1)
|
||||
self.assertEqual(await asyncio.wait_for(channel.receive(), 1), b'n' * 40)
|
||||
self.assertFalse(tasks[0].done())
|
||||
self.assertFalse(tasks[1].done())
|
||||
self.assertTrue(tasks[2].cancelled())
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
|
||||
async def test_capacity_rotation_never_retires_unretained_or_plaintext_request(self):
|
||||
release = asyncio.Event()
|
||||
async def handler(request):
|
||||
await request.extensions['trace']('http2.send_request_body.complete', {'stream_id': 3})
|
||||
await release.wait()
|
||||
return httpx.Response(200, content=b'r' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
with patch('proxy.cf_h2.MAX_CHANNEL_REQUESTS', 2), \
|
||||
patch('proxy.cf_h2.REPLAY_REQUEST_SECONDS', .005), \
|
||||
patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .002):
|
||||
await channel.send(b'\0' * 8 + b'p' * 32, False)
|
||||
await channel.send(b'samekey!' + b'u' * (64 * 1024), False)
|
||||
extra = asyncio.create_task(channel.send(b'samekey!' + b'n' * 32, False))
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
await asyncio.sleep(.04)
|
||||
self.assertFalse(extra.done())
|
||||
self.assertEqual(channel.capacity_rotations, 0)
|
||||
self.assertTrue(all(not task.done() for task in channel.pending))
|
||||
release.set()
|
||||
await asyncio.wait_for(extra, 1)
|
||||
finally:
|
||||
release.set()
|
||||
extra.cancel()
|
||||
recovery.cancel()
|
||||
await asyncio.gather(extra, recovery, return_exceptions=True)
|
||||
|
||||
async def test_http1_rejected_by_preflight(self):
|
||||
self._transport(lambda request: httpx.Response(200, extensions={'http_version': b'HTTP/1.1'}))
|
||||
with self.assertRaisesRegex(ValueError, 'HTTP/1.1'):
|
||||
await self.lane._preflight()
|
||||
|
||||
async def test_telegram_head_501_still_allows_http2_preflight(self):
|
||||
self._transport(lambda request: httpx.Response(501, extensions={'http_version': b'HTTP/2'}))
|
||||
await self.lane._preflight()
|
||||
|
||||
async def _packet_bridge(self, *, content_length=True, truncate=False, concurrent_small=False):
|
||||
first_chunk_read, release_tail = asyncio.Event(), asyncio.Event()
|
||||
body = b'x' * (64 * 1024 + 40)
|
||||
first_chunk = body[:16384]
|
||||
class ResponseStream(httpx.AsyncByteStream):
|
||||
async def __aiter__(self):
|
||||
yield first_chunk
|
||||
first_chunk_read.set()
|
||||
await release_tail.wait()
|
||||
if not truncate:
|
||||
yield body[len(first_chunk):]
|
||||
async def handler(request):
|
||||
if request.content == b's' * 40:
|
||||
return httpx.Response(200, content=b'z' * 40,
|
||||
extensions={'http_version': b'HTTP/2'})
|
||||
headers = {'content-length': str(len(body))} if content_length else {}
|
||||
return httpx.Response(200, headers=headers, stream=ResponseStream(),
|
||||
extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
class Writer:
|
||||
def __init__(self):
|
||||
self.body = bytearray()
|
||||
self.changed = asyncio.Event()
|
||||
def write(self, data):
|
||||
self.body.extend(data)
|
||||
self.changed.set()
|
||||
async def drain(self):
|
||||
await asyncio.sleep(0)
|
||||
reader, writer = asyncio.StreamReader(), Writer()
|
||||
channel = self._channel()
|
||||
ctx = SimpleNamespace(clt_enc=_IdentityCipher(), clt_dec=_IdentityCipher())
|
||||
task = asyncio.create_task(bridge_h2(reader, writer, channel, ctx, PROTO_TAG_INTERMEDIATE))
|
||||
reader.feed_data(struct.pack('<I', 40) + b'l' * 40)
|
||||
try:
|
||||
await asyncio.wait_for(first_chunk_read.wait(), 1)
|
||||
self.assertEqual(writer.body, b'')
|
||||
if concurrent_small:
|
||||
reader.feed_data(struct.pack('<I', 40) + b's' * 40)
|
||||
await asyncio.wait_for(writer.changed.wait(), 1)
|
||||
self.assertEqual(writer.body, struct.pack('<I', 40) + b'z' * 40)
|
||||
release_tail.set()
|
||||
if truncate:
|
||||
await asyncio.wait_for(task, 1)
|
||||
self.assertTrue(channel.closed)
|
||||
self.assertEqual(writer.body, b'')
|
||||
else:
|
||||
expected = struct.pack('<I', len(body)) + body
|
||||
if concurrent_small:
|
||||
expected = struct.pack('<I', 40) + b'z' * 40 + expected
|
||||
for _ in range(100):
|
||||
if len(writer.body) >= len(expected):
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
self.assertEqual(writer.body, expected)
|
||||
finally:
|
||||
release_tail.set()
|
||||
reader.feed_eof()
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
async def test_native_frame_starts_only_after_entire_body_is_available(self):
|
||||
await self._packet_bridge()
|
||||
|
||||
async def test_ready_reply_bypasses_incomplete_large_response_on_same_channel(self):
|
||||
await self._packet_bridge(concurrent_small=True)
|
||||
|
||||
async def test_response_without_length_uses_buffered_framing(self):
|
||||
await self._packet_bridge(content_length=False)
|
||||
|
||||
async def test_truncated_stream_closes_native_channel(self):
|
||||
await self._packet_bridge(truncate=True)
|
||||
|
||||
async def test_recovery_pauses_during_body_receive_and_resumes_afterwards(self):
|
||||
started, release = asyncio.Event(), asyncio.Event()
|
||||
calls = []
|
||||
|
||||
class SlowBody(httpx.AsyncByteStream):
|
||||
async def __aiter__(self):
|
||||
yield b'r' * 20
|
||||
started.set()
|
||||
await release.wait()
|
||||
yield b'r' * 20
|
||||
|
||||
def handler(request):
|
||||
calls.append(request.content)
|
||||
if len(calls) == 1:
|
||||
return httpx.Response(200, stream=SlowBody(), extensions={'http_version': b'HTTP/2'})
|
||||
return httpx.Response(200, content=b's' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
await channel.send(b'x' * 40, False)
|
||||
await asyncio.wait_for(started.wait(), 1)
|
||||
with patch('proxy.cf_h2.REPLAY_IDLE_SECONDS', .01), patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005):
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
await asyncio.sleep(.06)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(channel.receiving_http, 1)
|
||||
release.set()
|
||||
self.assertEqual(await asyncio.wait_for(channel.receive(), 1), b'r' * 40)
|
||||
self.assertEqual(await asyncio.wait_for(channel.receive(), 1), b's' * 40)
|
||||
self.assertEqual(calls, [b'x' * 40, b'x' * 40])
|
||||
self.assertEqual(channel.receiving_http, 0)
|
||||
self.assertEqual(self.lane.reply_buffer_bytes, 0)
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
|
||||
async def test_lane_reply_memory_bound_and_cancellation_cleanup(self):
|
||||
started = asyncio.Event()
|
||||
|
||||
class SlowBody(httpx.AsyncByteStream):
|
||||
async def __aiter__(self):
|
||||
yield b'r' * 32
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
def handler(request):
|
||||
if request.content == b'a' * 40:
|
||||
return httpx.Response(200, stream=SlowBody(), extensions={'http_version': b'HTTP/2'})
|
||||
return httpx.Response(200, content=b'r' * 16, extensions={'http_version': b'HTTP/2'})
|
||||
|
||||
self._transport(handler)
|
||||
slow = self._channel()
|
||||
with patch('proxy.cf_h2.MAX_LANE_REPLY_BYTES', 40):
|
||||
await slow.send(b'a' * 40, False)
|
||||
await asyncio.wait_for(started.wait(), 1)
|
||||
self.assertEqual(self.lane.reply_buffer_bytes, 32)
|
||||
with self.assertRaisesRegex(BufferError, 'response buffers'):
|
||||
await self.lane._post(b'b' * 40, 2)
|
||||
self.assertEqual(self.lane.reply_buffer_bytes, 32)
|
||||
self.assertEqual(self.lane.failed_until, 0)
|
||||
await slow.close()
|
||||
self.assertEqual(slow.receiving_http, 0)
|
||||
self.assertEqual(self.lane.reply_buffer_bytes, 0)
|
||||
self.assertEqual(await self.lane._post(b'b' * 40, 2), b'r' * 16)
|
||||
|
||||
async def test_native_disconnect_does_not_cool_down_healthy_lane(self):
|
||||
self._transport(lambda request: httpx.Response(
|
||||
200, content=b'r' * 65536, extensions={'http_version': b'HTTP/2'}))
|
||||
|
||||
class DisconnectedWriter:
|
||||
def write(self, data):
|
||||
pass
|
||||
|
||||
async def drain(self):
|
||||
raise ConnectionResetError('Connection lost')
|
||||
|
||||
channel = self._channel()
|
||||
reader = asyncio.StreamReader()
|
||||
reader.feed_data(struct.pack('<I', 40) + b'x' * 40)
|
||||
ctx = SimpleNamespace(clt_enc=_IdentityCipher(), clt_dec=_IdentityCipher())
|
||||
await asyncio.wait_for(bridge_h2(reader, DisconnectedWriter(), channel, ctx,
|
||||
PROTO_TAG_INTERMEDIATE), 1)
|
||||
self.assertTrue(channel.closed)
|
||||
self.assertEqual(self.lane.failed_until, 0)
|
||||
self.assertEqual(self.lane.errors, 0)
|
||||
healthy = self._channel()
|
||||
await healthy.send(b'h' * 40, False)
|
||||
self.assertEqual(await asyncio.wait_for(healthy.receive(), 1), b'r' * 65536)
|
||||
|
||||
async def test_slow_receiver_queue_overflow_does_not_break_other_channels(self):
|
||||
async def handler(request):
|
||||
return httpx.Response(200, content=request.content, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
slow, healthy = self._channel(), self._channel()
|
||||
with patch('proxy.cf_h2.MAX_CHANNEL_BYTES', 80):
|
||||
for _ in range(3):
|
||||
await slow.send(b's' * 40, False)
|
||||
await asyncio.gather(*list(slow.pending))
|
||||
with self.assertRaises(BufferError):
|
||||
await asyncio.wait_for(slow.receive(), 1)
|
||||
await healthy.send(b'h' * 40, False)
|
||||
self.assertEqual(await asyncio.wait_for(healthy.receive(), 1), b'h' * 40)
|
||||
self.assertFalse(self.lane.closed)
|
||||
|
||||
async def test_simultaneous_pool_opens_preflight_once_and_share_client(self):
|
||||
pool = CfH2Pool()
|
||||
balancer = _Balancer()
|
||||
balancer.update_domains_list(['example.org'])
|
||||
started = []
|
||||
async def preflight(lane):
|
||||
started.append(lane)
|
||||
await asyncio.sleep(0)
|
||||
try:
|
||||
with patch('proxy.cf_h2.balancer', balancer), patch.object(_HttpLane, '_preflight', preflight):
|
||||
channels = await asyncio.gather(*(pool.open(4, 'owned-test') for _ in range(6)))
|
||||
self.assertEqual(len(started), 1)
|
||||
self.assertEqual(len({id(channel.lane.client) for channel in channels}), 1)
|
||||
self.assertEqual(len({channel.channel_id for channel in channels}), 6)
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
async def test_recovery_reuses_exact_ciphertext_when_original_http_is_still_pending(self):
|
||||
calls = []
|
||||
original_waiting = asyncio.Event()
|
||||
body = b'opaque-key-and-message-id' + b'x' * 16
|
||||
async def handler(request):
|
||||
calls.append(request.content)
|
||||
if len(calls) == 1:
|
||||
original_waiting.set()
|
||||
await asyncio.Event().wait()
|
||||
return httpx.Response(200, content=b'r' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
await channel.send(body, False)
|
||||
await original_waiting.wait()
|
||||
with patch('proxy.cf_h2.REPLAY_REQUEST_SECONDS', .01), patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005):
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
self.assertEqual(await asyncio.wait_for(channel.receive(), 1), b'r' * 40)
|
||||
self.assertEqual(calls, [body, body])
|
||||
self.assertEqual(channel.replay_requests, 1)
|
||||
self.assertEqual(self.lane.max_inflight, 2)
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
await channel.close()
|
||||
self.assertFalse(channel.pending)
|
||||
self.assertEqual(self.lane.inflight, 0)
|
||||
|
||||
async def test_recovery_after_http_finished_without_native_reply(self):
|
||||
calls = []
|
||||
async def handler(request):
|
||||
calls.append(request.content)
|
||||
return httpx.Response(200, content=b'' if len(calls) == 1 else b'r' * 40,
|
||||
extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
body = b'k' * 40
|
||||
await channel.send(body, False)
|
||||
await asyncio.gather(*list(channel.pending))
|
||||
self.assertTrue(channel.queue.empty())
|
||||
with patch('proxy.cf_h2.REPLAY_IDLE_SECONDS', .01), patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005):
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
self.assertEqual(await asyncio.wait_for(channel.receive(), 1), b'r' * 40)
|
||||
await asyncio.sleep(.03)
|
||||
self.assertEqual(calls, [body, body])
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
|
||||
async def test_idle_receiver_preserves_capacity_for_new_native_request(self):
|
||||
calls = []
|
||||
probes_waiting = asyncio.Event()
|
||||
async def handler(request):
|
||||
calls.append(request.content)
|
||||
if calls.count(request.content) > 1:
|
||||
if len(calls) == 4:
|
||||
probes_waiting.set()
|
||||
await asyncio.Event().wait()
|
||||
return httpx.Response(200, content=b'r' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
bodies = [b'samekey!' + bytes([index]) * 32 for index in range(4)]
|
||||
for body in bodies[:3]:
|
||||
await channel.send(body, False)
|
||||
await asyncio.gather(*list(channel.pending))
|
||||
await channel.receive()
|
||||
with patch('proxy.cf_h2.REPLAY_IDLE_SECONDS', .01), patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005):
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
await asyncio.wait_for(probes_waiting.wait(), 1)
|
||||
await asyncio.sleep(.03)
|
||||
self.assertEqual(channel.replay_requests, 1)
|
||||
self.assertEqual(len(channel.replay_pending), 1)
|
||||
await asyncio.wait_for(channel.send(bodies[3], False), 1)
|
||||
self.assertEqual(await asyncio.wait_for(channel.receive(), 1), b'r' * 40)
|
||||
self.assertEqual(calls[3:4], [bodies[2]])
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
await asyncio.gather(channel.close(), channel.close())
|
||||
self.assertEqual(self.lane.inflight, 0)
|
||||
self.assertEqual(channel.pending_bytes, 0)
|
||||
self.assertFalse(channel.replay_pending)
|
||||
|
||||
async def test_handshake_and_expired_ciphertext_are_never_replayed(self):
|
||||
calls = []
|
||||
async def handler(request):
|
||||
calls.append(request.content)
|
||||
return httpx.Response(200, content=b'r' * 40, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
for body in (b'\x00' * 8 + b'h' * 32, b'k' * 40):
|
||||
await channel.send(body, False)
|
||||
await asyncio.gather(*list(channel.pending))
|
||||
await channel.receive()
|
||||
# Windows may give consecutive monotonic() calls the same timestamp;
|
||||
# an event wakeup need not advance the clock like the old sleep did.
|
||||
channel.replay_history[-1].sent_at -= 1
|
||||
with patch('proxy.cf_h2.REPLAY_IDLE_SECONDS', .01), patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005), \
|
||||
patch('proxy.cf_h2.REPLAY_MAX_AGE', 0):
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
await asyncio.sleep(.04)
|
||||
self.assertEqual(len(calls), 2)
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
|
||||
async def test_unhelpful_replies_do_not_cause_unbounded_recovery(self):
|
||||
calls = []
|
||||
retries_finished = asyncio.Event()
|
||||
async def handler(request):
|
||||
calls.append((request.content, time.monotonic()))
|
||||
if len(calls) == 5:
|
||||
retries_finished.set()
|
||||
return httpx.Response(200, content=b'', extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
bodies = [b'samekey!' + bytes([index]) * 32 for index in range(2)]
|
||||
for body in bodies:
|
||||
await channel.send(body, False)
|
||||
await asyncio.gather(*list(channel.pending))
|
||||
with patch('proxy.cf_h2.REPLAY_IDLE_SECONDS', .005), \
|
||||
patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .002), \
|
||||
patch('proxy.cf_h2.REPLAY_RETRY_SECONDS', .02):
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
await asyncio.wait_for(retries_finished.wait(), 1)
|
||||
await asyncio.sleep(.06)
|
||||
older = [when for body, when in calls if body == bodies[0]]
|
||||
latest = [when for body, when in calls if body == bodies[1]]
|
||||
# Completed history is not scanned; only the latest is polled.
|
||||
self.assertEqual(len(older), 1)
|
||||
self.assertEqual(len(latest), 4)
|
||||
for first, second in zip(latest[1:], latest[2:]):
|
||||
self.assertGreaterEqual(second - first, .015)
|
||||
self.assertFalse(channel.pending)
|
||||
self.assertFalse(channel.replay_pending)
|
||||
self.assertEqual(self.lane.inflight, 0)
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
|
||||
async def test_old_pending_request_has_bounded_retries_after_newer_requests(self):
|
||||
body = b'samekey!' + b't' * 32
|
||||
original_started = asyncio.Event()
|
||||
retries_finished = asyncio.Event()
|
||||
attempts = 0
|
||||
async def handler(request):
|
||||
nonlocal attempts
|
||||
if request.content == body:
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
original_started.set()
|
||||
await asyncio.Event().wait()
|
||||
if attempts == 4:
|
||||
retries_finished.set()
|
||||
return httpx.Response(200, content=b'', extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
await channel.send(body, False)
|
||||
await original_started.wait()
|
||||
original = next(iter(channel.pending))
|
||||
await channel.send(b'samekey!' + b'n' * 32, False)
|
||||
with patch('proxy.cf_h2.REPLAY_IDLE_SECONDS', .01), \
|
||||
patch('proxy.cf_h2.REPLAY_REQUEST_SECONDS', .02), \
|
||||
patch('proxy.cf_h2.REPLAY_CHECK_SECONDS', .005), \
|
||||
patch('proxy.cf_h2.REPLAY_RETRY_SECONDS', .02):
|
||||
recovery = asyncio.create_task(channel._recover())
|
||||
try:
|
||||
await asyncio.wait_for(retries_finished.wait(), 1)
|
||||
await asyncio.sleep(.06)
|
||||
self.assertEqual(attempts, 4)
|
||||
self.assertFalse(original.done())
|
||||
self.assertFalse(channel.replay_pending)
|
||||
finally:
|
||||
recovery.cancel()
|
||||
await asyncio.gather(recovery, return_exceptions=True)
|
||||
|
||||
async def test_closed_pool_cannot_open_new_channels(self):
|
||||
pool = CfH2Pool()
|
||||
await pool.close()
|
||||
with self.assertRaisesRegex(ConnectionError, 'pool closed'):
|
||||
await pool.open(4, 'owned-test')
|
||||
|
||||
async def test_concurrent_upload_and_recovery_cannot_exceed_channel_request_bound(self):
|
||||
release = asyncio.Event()
|
||||
async def handler(request):
|
||||
await release.wait()
|
||||
return httpx.Response(200, content=request.content, extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
for _ in range(MAX_CHANNEL_REQUESTS - 1):
|
||||
await channel.send(b'k' * 40, False)
|
||||
first = asyncio.create_task(channel.send(b'a' * 40, False))
|
||||
second = asyncio.create_task(channel.send(b'b' * 40, False))
|
||||
try:
|
||||
await asyncio.wait_for(first, 1)
|
||||
await asyncio.sleep(.02)
|
||||
self.assertEqual(len(channel.pending), MAX_CHANNEL_REQUESTS)
|
||||
self.assertFalse(second.done())
|
||||
release.set()
|
||||
await asyncio.wait_for(second, 1)
|
||||
finally:
|
||||
release.set()
|
||||
first.cancel()
|
||||
second.cancel()
|
||||
await asyncio.gather(first, second, return_exceptions=True)
|
||||
|
||||
async def test_failing_preflight_returns_to_ws_without_consuming_native_data(self):
|
||||
pool = CfH2Pool()
|
||||
balancer = _Balancer()
|
||||
balancer.update_domains_list(['example.org'])
|
||||
async def preflight(lane):
|
||||
raise OSError('owned connection failure')
|
||||
try:
|
||||
with patch('proxy.cf_h2.balancer', balancer), patch.object(_HttpLane, '_preflight', preflight):
|
||||
self.assertIsNone(await pool.open(4, 'owned-test'))
|
||||
self.assertFalse(pool.lanes)
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
async def test_hanging_setup_returns_to_ws_and_cleans_up_lane(self):
|
||||
pool = CfH2Pool()
|
||||
balancer = _Balancer()
|
||||
balancer.update_domains_list(['example.org'])
|
||||
lanes = []
|
||||
|
||||
async def preflight(lane):
|
||||
lanes.append(lane)
|
||||
await asyncio.Event().wait()
|
||||
|
||||
try:
|
||||
with patch('proxy.cf_h2.balancer', balancer), \
|
||||
patch('proxy.cf_h2.SETUP_TIMEOUT', .02), \
|
||||
patch.object(_HttpLane, '_preflight', preflight):
|
||||
self.assertIsNone(await pool.open(4, 'owned-test'))
|
||||
self.assertIsNone(await pool.open(4, 'owned-test'))
|
||||
self.assertEqual(len(lanes), 1)
|
||||
self.assertTrue(lanes[0].closed)
|
||||
self.assertTrue(lanes[0].client.is_closed)
|
||||
self.assertFalse(pool.lanes)
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
async def test_recovery_does_not_wait_for_channel_or_lane_capacity(self):
|
||||
channel = self._channel()
|
||||
body = b'k' * 40
|
||||
channel.replay_history.append(_ReplayPacket(0, body, 0))
|
||||
with patch('proxy.cf_h2.MAX_CHANNEL_REQUESTS', 0):
|
||||
self.assertFalse(await asyncio.wait_for(channel.send(body, False, replay=True), .1))
|
||||
with patch('proxy.cf_h2.MAX_LANE_REQUESTS', 0):
|
||||
self.assertFalse(await asyncio.wait_for(channel.send(body, False, replay=True), .1))
|
||||
self.assertFalse(channel.send_lock.locked())
|
||||
self.assertFalse(channel.replay_history[0].replayed)
|
||||
self.assertEqual(channel.requests, 0)
|
||||
self.assertEqual(self.lane.inflight, 0)
|
||||
|
||||
async def test_recovery_does_not_queue_behind_native_upload(self):
|
||||
channel = self._channel()
|
||||
async with channel.send_lock:
|
||||
self.assertFalse(await asyncio.wait_for(channel.send(b'k' * 40, False, replay=True), .1))
|
||||
self.assertEqual(channel.requests, 0)
|
||||
|
||||
async def test_wait_diagnostics_combine_channels_and_are_rate_limited(self):
|
||||
async def handler(request):
|
||||
await asyncio.Event().wait()
|
||||
self._transport(handler)
|
||||
now = time.monotonic()
|
||||
for _ in range(2):
|
||||
channel = self._channel()
|
||||
await channel.send(b'k' * 40, False)
|
||||
channel.last_progress = now - 4
|
||||
channel.pending_since = dict.fromkeys(channel.pending, now - 4)
|
||||
with self.assertLogs('tg-mtproto-proxy', level='DEBUG') as captured:
|
||||
for _ in range(20):
|
||||
self.lane.log_wait(now)
|
||||
self.assertEqual(len(captured.records), 1)
|
||||
self.assertIn('H2 WAIT', captured.output[0])
|
||||
self.assertIn('affected=2', captured.output[0])
|
||||
self.assertIn('channel=1', captured.output[0])
|
||||
self.assertIn('channel=2', captured.output[0])
|
||||
self.assertIn('recovery_left=', captured.output[0])
|
||||
|
||||
async def test_new_request_after_idle_does_not_log_a_stall(self):
|
||||
async def handler(request):
|
||||
await asyncio.Event().wait()
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
await channel.send(b'k' * 40, False)
|
||||
now = time.monotonic()
|
||||
channel.last_progress = now - 20
|
||||
with patch('proxy.cf_h2.log.isEnabledFor', return_value=True), \
|
||||
patch('proxy.cf_h2.log.debug') as debug:
|
||||
self.lane.log_wait(now)
|
||||
debug.assert_not_called()
|
||||
|
||||
async def test_wait_diagnostics_include_old_request_while_other_replies_progress(self):
|
||||
async def handler(request):
|
||||
await asyncio.Event().wait()
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
await channel.send(b'k' * 40, False)
|
||||
now = time.monotonic()
|
||||
channel.pending_since = dict.fromkeys(channel.pending, now - 4)
|
||||
channel.last_progress = now
|
||||
with self.assertLogs('tg-mtproto-proxy', level='DEBUG') as captured:
|
||||
self.lane.log_wait(now)
|
||||
self.assertEqual(len(captured.records), 1)
|
||||
self.assertIn('waiting_headers=1', captured.output[0])
|
||||
self.assertIn('idle_ms=0', captured.output[0])
|
||||
|
||||
async def test_slow_response_diagnostics_are_rate_limited(self):
|
||||
self._transport(lambda request: httpx.Response(
|
||||
200, content=b'r' * 40, extensions={'http_version': b'HTTP/2'}))
|
||||
with patch('proxy.cf_h2.SLOW_RESPONSE_SECONDS', 0), \
|
||||
self.assertLogs('tg-mtproto-proxy', level='DEBUG') as captured:
|
||||
for _ in range(10):
|
||||
await self.lane._post(b'k' * 40, 1)
|
||||
self.assertEqual(len(captured.records), 1)
|
||||
self.assertIn('H2 SLOW', captured.output[0])
|
||||
self.assertIn('headers_ms=', captured.output[0])
|
||||
self.assertIn('body_ms=', captured.output[0])
|
||||
|
||||
async def test_pending_details_link_replay_to_original_and_bound_history(self):
|
||||
calls = 0
|
||||
original_waiting = asyncio.Event()
|
||||
async def handler(request):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
original_waiting.set()
|
||||
await asyncio.Event().wait()
|
||||
return httpx.Response(200, content=b'', extensions={'http_version': b'HTTP/2'})
|
||||
self._transport(handler)
|
||||
channel = self._channel()
|
||||
with self.assertLogs('tg-mtproto-proxy', level='DEBUG') as captured:
|
||||
await channel.send(b'samekey!' + b'p' * 32, False)
|
||||
await original_waiting.wait()
|
||||
original = next(iter(channel.pending))
|
||||
with patch('proxy.cf_h2.REPLAY_REQUEST_SECONDS', 0):
|
||||
await channel.send(b'samekey!' + b'p' * 32, False, replay=True)
|
||||
await asyncio.gather(*(task for task in channel.pending if task is not original))
|
||||
trace = channel.request_traces[original]
|
||||
replay = channel.recent_traces[-1]
|
||||
self.assertEqual(replay.packet, trace.packet)
|
||||
self.assertNotEqual(replay.request, trace.request)
|
||||
self.assertTrue(replay.replay)
|
||||
now = time.monotonic()
|
||||
trace.started = now - 4
|
||||
channel.pending_since[original] = now - 4
|
||||
self.lane.log_wait(now)
|
||||
for index in range(24):
|
||||
await channel.send(b'samekey!' + bytes([index]) * 32, False)
|
||||
await asyncio.gather(*(task for task in channel.pending if task is not original))
|
||||
self.assertEqual(len(channel.recent_traces), 16)
|
||||
self.assertEqual(len(channel.request_traces), 1)
|
||||
output = '\n'.join(captured.output)
|
||||
self.assertIn('H2 DETAIL', output)
|
||||
self.assertIn('pkt=1', output)
|
||||
self.assertIn('replay=1 phase=complete', output)
|
||||
self.assertNotIn('samekey!', output)
|
||||
Reference in new issue
Block a user