relay_client.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. from __future__ import annotations
  2. import asyncio
  3. import contextlib
  4. import json
  5. import random
  6. import socket
  7. from dataclasses import dataclass
  8. import time
  9. from typing import Awaitable, Callable, Dict
  10. from .config import Config, RelayNode
  11. from .protocol import AUTH, PING, PONG, STATUS_OK, TCP_CLOSE, Frame, encode_json, read_frame, write_frame
  12. from .scheduler import Scheduler
  13. FrameHandler = Callable[["RelayConnection", Frame], Awaitable[None]]
  14. @dataclass
  15. class RelayConnection:
  16. node: RelayNode
  17. manager: "RelayManager"
  18. reader: asyncio.StreamReader
  19. writer: asyncio.StreamWriter
  20. closed: bool = False
  21. handlers: Dict[tuple[int, int], FrameHandler] = None
  22. dispatch_tasks: Dict[tuple[int, int], asyncio.Task] = None
  23. pump_task: asyncio.Task | None = None
  24. keepalive_task: asyncio.Task | None = None
  25. last_pong_at: float = 0.0
  26. send_lock: asyncio.Lock | None = None
  27. closed_event: asyncio.Event | None = None
  28. dropped_frames: Dict[int, int] = None
  29. dropped_report_task: asyncio.Task | None = None
  30. supports_tcp: bool = True
  31. supports_udp: bool = True
  32. def __post_init__(self) -> None:
  33. if self.handlers is None:
  34. self.handlers = {}
  35. if self.dispatch_tasks is None:
  36. self.dispatch_tasks = {}
  37. if self.send_lock is None:
  38. self.send_lock = asyncio.Lock()
  39. if self.closed_event is None:
  40. self.closed_event = asyncio.Event()
  41. if self.dropped_frames is None:
  42. self.dropped_frames = {}
  43. async def start(self) -> None:
  44. print(f"[edge] connecting relay name={self.node.name} addr={self.node.host}:{self.node.port}")
  45. await write_frame(self.writer, Frame(AUTH, 0, 0, 0, 0, encode_json({"token": self.node.token})))
  46. frame = await read_frame(self.reader)
  47. if frame.kind != AUTH or frame.packet_id != STATUS_OK:
  48. raise ConnectionError(f"relay auth failed: {self.node.name}")
  49. ack = {}
  50. if frame.payload:
  51. try:
  52. ack = json.loads(frame.payload.decode("utf-8"))
  53. except Exception:
  54. ack = {}
  55. self.supports_tcp = not bool(ack.get("udp_only", False))
  56. self.supports_udp = True
  57. self.last_pong_at = time.monotonic()
  58. self.keepalive_task = asyncio.create_task(self._keepalive())
  59. self.pump_task = asyncio.create_task(self._pump())
  60. print(
  61. f"[edge] relay connected name={self.node.name} addr={self.node.host}:{self.node.port} "
  62. f"supports_tcp={self.supports_tcp} supports_udp={self.supports_udp}"
  63. )
  64. async def _keepalive(self) -> None:
  65. try:
  66. while not self.closed:
  67. await asyncio.sleep(self.manager.config.relay_ping_interval)
  68. if self.closed:
  69. break
  70. if self.last_pong_at and time.monotonic() - self.last_pong_at > (self.manager.config.relay_ping_interval + self.manager.config.relay_ping_timeout):
  71. print(f"[edge] relay health timeout name={self.node.name} addr={self.node.host}:{self.node.port} timeout={self.manager.config.relay_ping_timeout}")
  72. await self.close()
  73. break
  74. await self.send(Frame(PING, 0, 0, 0, 0, b""))
  75. except asyncio.CancelledError:
  76. pass
  77. except Exception:
  78. await self.close()
  79. async def _pump(self) -> None:
  80. try:
  81. while True:
  82. frame = await read_frame(self.reader)
  83. if frame.kind == PONG:
  84. self.last_pong_at = time.monotonic()
  85. continue
  86. handler = self.handlers.get((frame.session_id, frame.stream_id))
  87. if handler:
  88. self._dispatch_frame(frame, handler)
  89. else:
  90. self._record_dropped_frame(frame.kind)
  91. except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError, OSError):
  92. pass
  93. except Exception:
  94. pass
  95. finally:
  96. await self.close()
  97. def _dispatch_frame(self, frame: Frame, handler: FrameHandler) -> None:
  98. key = (frame.session_id, frame.stream_id)
  99. previous = self.dispatch_tasks.get(key)
  100. task = asyncio.create_task(self._run_handler(key, frame, handler, previous))
  101. self.dispatch_tasks[key] = task
  102. async def _run_handler(self, key: tuple[int, int], frame: Frame, handler: FrameHandler, previous: asyncio.Task | None) -> None:
  103. try:
  104. if previous is not None:
  105. with contextlib.suppress(Exception):
  106. await previous
  107. if self.closed:
  108. return
  109. await handler(self, frame)
  110. except asyncio.CancelledError:
  111. pass
  112. except Exception:
  113. if not self.closed:
  114. await self.close()
  115. finally:
  116. if self.dispatch_tasks.get(key) is asyncio.current_task():
  117. self.dispatch_tasks.pop(key, None)
  118. def _record_dropped_frame(self, kind: int) -> None:
  119. self.dropped_frames[kind] = self.dropped_frames.get(kind, 0) + 1
  120. if self.dropped_report_task is None or self.dropped_report_task.done():
  121. self.dropped_report_task = asyncio.create_task(self._report_dropped_frames())
  122. async def _report_dropped_frames(self) -> None:
  123. try:
  124. await asyncio.sleep(5)
  125. dropped = self.dropped_frames
  126. self.dropped_frames = {}
  127. if dropped:
  128. detail = ", ".join(f"kind={kind} count={count}" for kind, count in sorted(dropped.items()))
  129. print(f"[edge] relay frame dropped summary name={self.node.name} {detail}")
  130. except asyncio.CancelledError:
  131. pass
  132. async def send(self, frame: Frame) -> None:
  133. if self.closed:
  134. raise ConnectionError(f"relay closed: {self.node.name}")
  135. assert self.send_lock is not None
  136. async with self.send_lock:
  137. if self.closed:
  138. raise ConnectionError(f"relay closed: {self.node.name}")
  139. await write_frame(self.writer, frame)
  140. def bind(self, session_id: int, stream_id: int, handler: FrameHandler) -> None:
  141. self.handlers[(session_id, stream_id)] = handler
  142. def unbind(self, session_id: int, stream_id: int) -> None:
  143. self.handlers.pop((session_id, stream_id), None)
  144. task = self.dispatch_tasks.pop((session_id, stream_id), None)
  145. if task is not None:
  146. task.cancel()
  147. async def close(self) -> None:
  148. if self.closed:
  149. return
  150. self.closed = True
  151. assert self.closed_event is not None
  152. self.closed_event.set()
  153. handlers = list(self.handlers.items())
  154. self.handlers.clear()
  155. dispatch_tasks = list(self.dispatch_tasks.values())
  156. self.dispatch_tasks.clear()
  157. self.manager.on_closed(self)
  158. for (session_id, stream_id), handler in handlers:
  159. with contextlib.suppress(Exception):
  160. await handler(self, Frame(TCP_CLOSE, session_id, stream_id, 0, 0, b""))
  161. for task in dispatch_tasks:
  162. task.cancel()
  163. for task in dispatch_tasks:
  164. with contextlib.suppress(Exception):
  165. await task
  166. if self.dropped_report_task and self.dropped_report_task is not asyncio.current_task():
  167. self.dropped_report_task.cancel()
  168. with contextlib.suppress(Exception):
  169. await self.dropped_report_task
  170. if self.pump_task and self.pump_task is not asyncio.current_task():
  171. self.pump_task.cancel()
  172. with contextlib.suppress(Exception):
  173. await self.pump_task
  174. if self.keepalive_task and self.keepalive_task is not asyncio.current_task():
  175. self.keepalive_task.cancel()
  176. with contextlib.suppress(Exception):
  177. await self.keepalive_task
  178. self.writer.close()
  179. with contextlib.suppress(Exception):
  180. await self.writer.wait_closed()
  181. class RelayManager:
  182. def __init__(self, config: Config) -> None:
  183. self.config = config
  184. self.scheduler = Scheduler(config)
  185. self.connections: Dict[str, RelayConnection] = {}
  186. self.tasks: list[asyncio.Task] = []
  187. async def start(self) -> None:
  188. await self.scheduler.start()
  189. for node in self.config.relays:
  190. self.tasks.append(asyncio.create_task(self._maintain(node)))
  191. async def _maintain(self, node: RelayNode) -> None:
  192. backoff = self.config.relay_reconnect_delay
  193. while True:
  194. current = self.connections.get(node.name)
  195. if current is not None and not current.closed:
  196. assert current.closed_event is not None
  197. await current.closed_event.wait()
  198. continue
  199. attempt = 1
  200. while True:
  201. try:
  202. print(f"[edge] relay reconnect attempt name={node.name} addr={node.host}:{node.port} attempt={attempt} backoff={backoff:.1f}s")
  203. reader, writer = await asyncio.wait_for(asyncio.open_connection(node.host, node.port), timeout=self.config.relay_open_timeout)
  204. connection = RelayConnection(node=node, manager=self, reader=reader, writer=writer)
  205. sock = writer.get_extra_info("socket")
  206. if sock is not None and self.config.relay_tcp_nodelay:
  207. with contextlib.suppress(OSError):
  208. sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
  209. await connection.start()
  210. self.connections[node.name] = connection
  211. backoff = self.config.relay_reconnect_delay
  212. assert connection.closed_event is not None
  213. await connection.closed_event.wait()
  214. print(f"[edge] relay supervisor noticed close name={node.name} addr={node.host}:{node.port}")
  215. break
  216. except asyncio.CancelledError:
  217. raise
  218. except Exception as exc:
  219. print(f"[edge] relay connect failed name={node.name} addr={node.host}:{node.port} attempt={attempt} error={exc!r}")
  220. jitter = random.uniform(0, min(1.0, backoff * 0.2))
  221. await asyncio.sleep(backoff + jitter)
  222. backoff = min(self.config.relay_reconnect_max_delay, max(self.config.relay_reconnect_delay, backoff * 2))
  223. attempt += 1
  224. def on_closed(self, connection: RelayConnection) -> None:
  225. current = self.connections.get(connection.node.name)
  226. if current is connection:
  227. self.connections.pop(connection.node.name, None)
  228. def available(self) -> list[RelayConnection]:
  229. chosen = {node.name for node in self.scheduler.choose()}
  230. preferred = [self.connections[name] for name in chosen if name in self.connections and not self.connections[name].closed and self.connections[name].supports_tcp]
  231. if preferred:
  232. return preferred
  233. return [conn for conn in self.connections.values() if not conn.closed and conn.supports_tcp]
  234. def available_udp(self) -> list[RelayConnection]:
  235. chosen = {node.name for node in self.scheduler.choose()}
  236. preferred = [self.connections[name] for name in chosen if name in self.connections and not self.connections[name].closed and self.connections[name].supports_udp]
  237. if preferred:
  238. return preferred
  239. return [conn for conn in self.connections.values() if not conn.closed and conn.supports_udp]
  240. def snapshot(self) -> list[dict[str, object]]:
  241. data = self.scheduler.snapshot()
  242. online = {name for name, conn in self.connections.items() if not conn.closed}
  243. for item in data:
  244. item["online"] = item["name"] in online
  245. return data