Correct handle of the 404 mtproto error

proxy/cf_h2.py

@@ -79,6 +79,14 @@ class _ReplyBufferFull(BufferError):
     """A local memory limit"""
 
 
+class _MTProtoTransportError(Exception):
+    """An endpoint error that the native client must receive, not just EOF."""
+
+    def __init__(self, code: int, source: str):
+        super().__init__('%d (%s)' % (code, source))
+        self.code = code
+
+
 @dataclass
 class _ReplayPacket:
     sent_at: float
@@ -139,6 +147,7 @@ class _HttpChannel:
     def __init__(self, lane, channel_id: int, label: str):
         self.lane, self.channel_id, self.label = lane, channel_id, label
         self.closed = False
+        self.transport_error: Optional[int] = None
         self.pending: Dict[asyncio.Task, int] = {}
         self.pending_since: Dict[asyncio.Task, float] = {}
         self.sent_since: Dict[asyncio.Task, float] = {}
@@ -336,6 +345,8 @@ class _HttpChannel:
     def _fail(self, error: Exception) -> None:
         if self.closed:
             return
+        if isinstance(error, _MTProtoTransportError):
+            self.transport_error = error.code
         self.closed = True
         self.recovery_wakeup.set()
         while not self.queue.empty():
@@ -460,14 +471,27 @@ class _HttpChannel:
             if trace is not None:
                 trace.phase = 'cancelled-' + trace.phase
             raise
+        except _MTProtoTransportError as exc:
+            if trace is not None:
+                trace.phase = 'transport-error'
+            stats.h2_errors += 1
+            if not self.closed:
+                log.warning('[%s] H2 lane=%d channel=%d host=%s MTProto transport error=%s '
+                            'request=%d encrypted=%d bytes=%d replay=%d; reporting to client',
+                            self.label, self.lane.lane_id, self.channel_id, self.lane.host, exc,
+                            trace.request if trace is not None else 0,
+                            body[:8] != b'\x00' * 8, len(body), replay)
+            self._fail(exc)
         except Exception as exc:
             if trace is not None:
                 trace.phase = 'error-' + type(exc).__name__
             stats.h2_errors += 1
             if isinstance(exc, (httpx.HTTPError, OSError, ValueError, BufferError)):
                 log.warning('[%s] H2 lane=%d channel=%d request failed: %s: %s; '
+                            'detail=[%s]; '
                             'closing native channel, reconnect may select another domain',
-                            self.label, self.lane.lane_id, self.channel_id, type(exc).__name__, exc)
+                            self.label, self.lane.lane_id, self.channel_id, type(exc).__name__, exc,
+                            trace.summary(time.monotonic()) if trace is not None else '-')
             else:
                 log.exception('[%s] H2 lane=%d channel=%d unexpected request failure',
                               self.label, self.lane.lane_id, self.channel_id)
@@ -610,8 +634,7 @@ class _HttpLane:
                                      extensions={'trace': self._trace}) as response:
             if response.http_version != 'HTTP/2':
                 raise ValueError('CF /api preflight negotiated ' + response.http_version)
-            if response.status_code in (403, 429) or (response.status_code >= 500
-                                                     and response.status_code != 501):
+            if not (200 <= response.status_code < 300 or response.status_code in (405, 501)):
                 raise ConnectionError('CF /api preflight HTTP %d cf_ray=%s' % (
                     response.status_code, response.headers.get('cf-ray', '-')))
             log.debug('H2 lane=%d ready host=%s status=%d ms=%.0f',
@@ -696,6 +719,9 @@ class _HttpLane:
                 self.max_headers_seconds = max(self.max_headers_seconds, headers_at - started)
                 if response.http_version != 'HTTP/2':
                     raise ValueError('CF /api downgraded to ' + response.http_version)
+                if response.status_code in (403, 404, 429, 444):
+                    raise _MTProtoTransportError(-response.status_code, 'HTTP %d cf_ray=%s' % (
+                        response.status_code, response.headers.get('cf-ray', '-')))
                 if response.status_code != 200:
                     raise ConnectionError('CF /api HTTP %d cf_ray=%s' % (
                         response.status_code, response.headers.get('cf-ray', '-')))
@@ -727,8 +753,9 @@ class _HttpLane:
                 if content and len(content) % 4:
                     raise ValueError('CF /api response is not an aligned MTProto packet')
                 if len(content) == 4:
-                    log.warning('H2 lane=%d channel=%d request=%d MTProto transport error=%d',
-                                self.lane_id, channel_id, request_id, struct.unpack('<i', content)[0])
+                    code, = struct.unpack('<i', content)
+                    if code < 0:
+                        raise _MTProtoTransportError(code, 'HTTP 200 payload')
                 self.responses += 1
                 if channel is not None:
                     channel.http_bytes += received
@@ -750,6 +777,9 @@ class _HttpLane:
                               (headers_at - started) * 1000, body_seconds * 1000,
                               trace.summary(finished) if trace is not None else '-')
                 return bytes(content)
+        except _MTProtoTransportError:
+            self.errors += 1
+            raise
         except _ReplyBufferFull:
             raise
         except (httpx.HTTPError, OSError, ValueError, BufferError):
@@ -930,19 +960,26 @@ async def bridge_h2(reader, writer, channel: _HttpChannel, ctx, tag: bytes) -> N
     tasks = [asyncio.create_task(upload()), asyncio.create_task(download()),
              asyncio.create_task(channel._recover())]
     try:
-        done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
-        for task in done:
-            task.result()
+        try:
+            done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+        finally:
+            for task in tasks:
+                task.cancel()
+            await asyncio.gather(*tasks, return_exceptions=True)
+        if channel.transport_error is not None:
+            await write_native(_encode_reply(struct.pack('<i', channel.transport_error), tag))
+            channel.down += 4
+            stats.bytes_down += 4
+        else:
+            for task in done:
+                task.result()
     except asyncio.IncompleteReadError:
         log.debug('[%s] H2 channel=%d native EOF', channel.label, channel.channel_id)
     except _NativeDeliveryError as exc:
         log.debug('[%s] H2 channel=%d native closed: %s', channel.label, channel.channel_id, exc)
-    except (OSError, ValueError, BufferError, asyncio.TimeoutError) as exc:
+    except (httpx.HTTPError, OSError, ValueError, BufferError, asyncio.TimeoutError) as exc:
         if not channel.closed:
             log.warning('[%s] H2 channel=%d bridge ended: %s: %s',
                         channel.label, channel.channel_id, type(exc).__name__, exc)
     finally:
-        for task in tasks:
-            task.cancel()
-        await asyncio.gather(*tasks, return_exceptions=True)
         await channel.close()