feat(telegram): auto-discover fallback IPs via DoH when api.telegram.org is unreachable (#3376)
* feat(telegram): auto-discover fallback IPs via DoH when api.telegram.org is unreachable On some networks (university, corporate), api.telegram.org resolves to a valid Telegram IP that is unreachable due to routing/firewall rules. A different IP in the same Telegram-owned 149.154.160.0/20 block works fine. This adds automatic fallback IP discovery at connect time: 1. Query Google and Cloudflare DNS-over-HTTPS for api.telegram.org A records 2. Exclude the system-DNS IP (the unreachable one), use the rest as fallbacks 3. If DoH is also blocked, fall back to a seed list (149.154.167.220) 4. TelegramFallbackTransport tries primary first, sticks to whichever works No configuration needed — works automatically. TELEGRAM_FALLBACK_IPS env var still available as manual override. Zero impact on healthy networks (primary path succeeds on first attempt, fallback never exercised). No new dependencies (uses httpx already in deps + stdlib socket). * fix: share transport instance and downgrade seed fallback log to info - Use single TelegramFallbackTransport shared between request and get_updates_request so sticky IP is shared across polling and API calls - Keep separate HTTPXRequest instances (different timeout settings) - Downgrade "using seed fallback IPs" from warning to info to avoid noisy logs on healthy networks * fix: add telegram.request mock and discovery fixture to remaining test files The original PR missed test_dm_topics.py and test_telegram_network_reconnect.py — both need the telegram.request mock module. The reconnect test also needs _no_auto_discovery since _handle_polling_network_error calls connect() which now invokes discover_fallback_ips(). --------- Co-authored-by: Mohan Qiao <Gavin-Qiao@users.noreply.github.com>
This commit is contained in:
Teknium
@@ -601,6 +601,14 @@ def _apply_env_overrides(config: GatewayConfig) -> None:
|
||||
config.platforms[Platform.TELEGRAM] = PlatformConfig()
|
||||
config.platforms[Platform.TELEGRAM].reply_to_mode = telegram_reply_mode
|
||||
|
||||
telegram_fallback_ips = os.getenv("TELEGRAM_FALLBACK_IPS", "")
|
||||
if telegram_fallback_ips:
|
||||
if Platform.TELEGRAM not in config.platforms:
|
||||
config.platforms[Platform.TELEGRAM] = PlatformConfig()
|
||||
config.platforms[Platform.TELEGRAM].extra["fallback_ips"] = [
|
||||
ip.strip() for ip in telegram_fallback_ips.split(",") if ip.strip()
|
||||
]
|
||||
|
||||
telegram_home = os.getenv("TELEGRAM_HOME_CHANNEL")
|
||||
if telegram_home and Platform.TELEGRAM in config.platforms:
|
||||
config.platforms[Platform.TELEGRAM].home_channel = HomeChannel(
|
||||
|
||||
@@ -11,7 +11,7 @@ import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, Optional, Any
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -25,6 +25,7 @@ try:
|
||||
filters,
|
||||
)
|
||||
from telegram.constants import ParseMode, ChatType
|
||||
from telegram.request import HTTPXRequest
|
||||
TELEGRAM_AVAILABLE = True
|
||||
except ImportError:
|
||||
TELEGRAM_AVAILABLE = False
|
||||
@@ -34,6 +35,7 @@ except ImportError:
|
||||
Application = Any
|
||||
CommandHandler = Any
|
||||
TelegramMessageHandler = Any
|
||||
HTTPXRequest = Any
|
||||
filters = None
|
||||
ParseMode = None
|
||||
ChatType = None
|
||||
@@ -59,6 +61,11 @@ from gateway.platforms.base import (
|
||||
cache_document_from_bytes,
|
||||
SUPPORTED_DOCUMENT_TYPES,
|
||||
)
|
||||
from gateway.platforms.telegram_network import (
|
||||
TelegramFallbackTransport,
|
||||
discover_fallback_ips,
|
||||
parse_fallback_ip_env,
|
||||
)
|
||||
|
||||
|
||||
def check_telegram_requirements() -> bool:
|
||||
@@ -138,6 +145,13 @@ class TelegramAdapter(BasePlatformAdapter):
|
||||
# DM Topics config from extra.dm_topics
|
||||
self._dm_topics_config: List[Dict[str, Any]] = self.config.extra.get("dm_topics", [])
|
||||
|
||||
def _fallback_ips(self) -> list[str]:
|
||||
"""Return validated fallback IPs from config (populated by _apply_env_overrides)."""
|
||||
configured = self.config.extra.get("fallback_ips", []) if getattr(self.config, "extra", None) else []
|
||||
if isinstance(configured, str):
|
||||
configured = configured.split(",")
|
||||
return parse_fallback_ip_env(",".join(str(v) for v in configured) if configured else None)
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_polling_conflict(error: Exception) -> bool:
|
||||
text = str(error).lower()
|
||||
@@ -474,7 +488,26 @@ class TelegramAdapter(BasePlatformAdapter):
|
||||
return False
|
||||
|
||||
# Build the application
|
||||
self._app = Application.builder().token(self.config.token).build()
|
||||
builder = Application.builder().token(self.config.token)
|
||||
fallback_ips = self._fallback_ips()
|
||||
if not fallback_ips:
|
||||
fallback_ips = await discover_fallback_ips()
|
||||
logger.info(
|
||||
"[%s] Auto-discovered Telegram fallback IPs: %s",
|
||||
self.name,
|
||||
", ".join(fallback_ips),
|
||||
)
|
||||
if fallback_ips:
|
||||
logger.warning(
|
||||
"[%s] Telegram fallback IPs active: %s",
|
||||
self.name,
|
||||
", ".join(fallback_ips),
|
||||
)
|
||||
transport = TelegramFallbackTransport(fallback_ips)
|
||||
request = HTTPXRequest(httpx_kwargs={"transport": transport})
|
||||
get_updates_request = HTTPXRequest(httpx_kwargs={"transport": transport})
|
||||
builder = builder.request(request).get_updates_request(get_updates_request)
|
||||
self._app = builder.build()
|
||||
self._bot = self._app.bot
|
||||
|
||||
# Register handlers
|
||||
|
||||
233
gateway/platforms/telegram_network.py
Normal file
233
gateway/platforms/telegram_network.py
Normal file
@@ -0,0 +1,233 @@
|
||||
"""Telegram-specific network helpers.
|
||||
|
||||
Provides a hostname-preserving fallback transport for networks where
|
||||
api.telegram.org resolves to an endpoint that is unreachable from the current
|
||||
host. The transport keeps the logical request host and TLS SNI as
|
||||
api.telegram.org while retrying the TCP connection against one or more fallback
|
||||
IPv4 addresses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import logging
|
||||
import socket
|
||||
from typing import Iterable, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TELEGRAM_API_HOST = "api.telegram.org"
|
||||
|
||||
# DNS-over-HTTPS providers used to discover Telegram API IPs that may differ
|
||||
# from the (potentially unreachable) IP returned by the local system resolver.
|
||||
_DOH_TIMEOUT = 4.0 # seconds — bounded so connect() isn't noticeably delayed
|
||||
|
||||
_DOH_PROVIDERS: list[dict] = [
|
||||
{
|
||||
"url": "https://dns.google/resolve",
|
||||
"params": {"name": _TELEGRAM_API_HOST, "type": "A"},
|
||||
"headers": {},
|
||||
},
|
||||
{
|
||||
"url": "https://cloudflare-dns.com/dns-query",
|
||||
"params": {"name": _TELEGRAM_API_HOST, "type": "A"},
|
||||
"headers": {"Accept": "application/dns-json"},
|
||||
},
|
||||
]
|
||||
|
||||
# Last-resort IPs when DoH is also blocked. These are stable Telegram Bot API
|
||||
# endpoints in the 149.154.160.0/20 block (same seed used by OpenClaw).
|
||||
_SEED_FALLBACK_IPS: list[str] = ["149.154.167.220"]
|
||||
|
||||
|
||||
class TelegramFallbackTransport(httpx.AsyncBaseTransport):
|
||||
"""Retry Telegram Bot API requests via fallback IPs while preserving TLS/SNI.
|
||||
|
||||
Requests continue to target https://api.telegram.org/... logically, but on
|
||||
connect failures the underlying TCP connection is retried against a known
|
||||
reachable IP. This is effectively the programmatic equivalent of
|
||||
``curl --resolve api.telegram.org:443:<ip>``.
|
||||
"""
|
||||
|
||||
def __init__(self, fallback_ips: Iterable[str], **transport_kwargs):
|
||||
self._fallback_ips = [ip for ip in dict.fromkeys(_normalize_fallback_ips(fallback_ips))]
|
||||
self._primary = httpx.AsyncHTTPTransport(**transport_kwargs)
|
||||
self._fallbacks = {
|
||||
ip: httpx.AsyncHTTPTransport(**transport_kwargs) for ip in self._fallback_ips
|
||||
}
|
||||
self._sticky_ip: Optional[str] = None
|
||||
self._sticky_lock = asyncio.Lock()
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host != _TELEGRAM_API_HOST or not self._fallback_ips:
|
||||
return await self._primary.handle_async_request(request)
|
||||
|
||||
sticky_ip = self._sticky_ip
|
||||
attempt_order: list[Optional[str]] = [sticky_ip] if sticky_ip else [None]
|
||||
for ip in self._fallback_ips:
|
||||
if ip != sticky_ip:
|
||||
attempt_order.append(ip)
|
||||
|
||||
last_error: Exception | None = None
|
||||
for ip in attempt_order:
|
||||
candidate = request if ip is None else _rewrite_request_for_ip(request, ip)
|
||||
transport = self._primary if ip is None else self._fallbacks[ip]
|
||||
try:
|
||||
response = await transport.handle_async_request(candidate)
|
||||
if ip is not None and self._sticky_ip != ip:
|
||||
async with self._sticky_lock:
|
||||
if self._sticky_ip != ip:
|
||||
self._sticky_ip = ip
|
||||
logger.warning(
|
||||
"[Telegram] Primary api.telegram.org path unreachable; using sticky fallback IP %s",
|
||||
ip,
|
||||
)
|
||||
return response
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
if not _is_retryable_connect_error(exc):
|
||||
raise
|
||||
if ip is None:
|
||||
logger.warning(
|
||||
"[Telegram] Primary api.telegram.org connection failed (%s); trying fallback IPs %s",
|
||||
exc,
|
||||
", ".join(self._fallback_ips),
|
||||
)
|
||||
continue
|
||||
logger.warning("[Telegram] Fallback IP %s failed: %s", ip, exc)
|
||||
continue
|
||||
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._primary.aclose()
|
||||
for transport in self._fallbacks.values():
|
||||
await transport.aclose()
|
||||
|
||||
|
||||
def _normalize_fallback_ips(values: Iterable[str]) -> list[str]:
|
||||
normalized: list[str] = []
|
||||
for value in values:
|
||||
raw = str(value).strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
addr = ipaddress.ip_address(raw)
|
||||
except ValueError:
|
||||
logger.warning("Ignoring invalid Telegram fallback IP: %r", raw)
|
||||
continue
|
||||
if addr.version != 4:
|
||||
logger.warning("Ignoring non-IPv4 Telegram fallback IP: %s", raw)
|
||||
continue
|
||||
normalized.append(str(addr))
|
||||
return normalized
|
||||
|
||||
|
||||
def parse_fallback_ip_env(value: str | None) -> list[str]:
|
||||
if not value:
|
||||
return []
|
||||
parts = [part.strip() for part in value.split(",")]
|
||||
return _normalize_fallback_ips(parts)
|
||||
|
||||
|
||||
def _resolve_system_dns() -> set[str]:
|
||||
"""Return the IPv4 addresses that the OS resolver gives for api.telegram.org."""
|
||||
try:
|
||||
results = socket.getaddrinfo(_TELEGRAM_API_HOST, 443, socket.AF_INET)
|
||||
return {addr[4][0] for addr in results}
|
||||
except Exception:
|
||||
return set()
|
||||
|
||||
|
||||
async def _query_doh_provider(
|
||||
client: httpx.AsyncClient, provider: dict
|
||||
) -> list[str]:
|
||||
"""Query one DoH provider and return A-record IPs."""
|
||||
try:
|
||||
resp = await client.get(
|
||||
provider["url"], params=provider["params"], headers=provider["headers"]
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
ips: list[str] = []
|
||||
for answer in data.get("Answer", []):
|
||||
if answer.get("type") != 1: # A record
|
||||
continue
|
||||
raw = answer.get("data", "").strip()
|
||||
try:
|
||||
ipaddress.ip_address(raw)
|
||||
ips.append(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
return ips
|
||||
except Exception as exc:
|
||||
logger.debug("DoH query to %s failed: %s", provider["url"], exc)
|
||||
return []
|
||||
|
||||
|
||||
async def discover_fallback_ips() -> list[str]:
|
||||
"""Auto-discover Telegram API IPs via DNS-over-HTTPS.
|
||||
|
||||
Resolves api.telegram.org through Google and Cloudflare DoH, collects all
|
||||
unique IPs, and excludes the system-DNS-resolved IP (which is presumably
|
||||
unreachable on this network). Falls back to a hardcoded seed list when DoH
|
||||
is also unavailable.
|
||||
"""
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(_DOH_TIMEOUT)) as client:
|
||||
doh_tasks = [_query_doh_provider(client, p) for p in _DOH_PROVIDERS]
|
||||
system_dns_task = asyncio.to_thread(_resolve_system_dns)
|
||||
results = await asyncio.gather(system_dns_task, *doh_tasks, return_exceptions=True)
|
||||
|
||||
# results[0] = system DNS IPs (set), results[1:] = DoH IP lists
|
||||
system_ips: set[str] = results[0] if isinstance(results[0], set) else set()
|
||||
|
||||
doh_ips: list[str] = []
|
||||
for r in results[1:]:
|
||||
if isinstance(r, list):
|
||||
doh_ips.extend(r)
|
||||
|
||||
# Deduplicate preserving order, exclude system-DNS IPs
|
||||
seen: set[str] = set()
|
||||
candidates: list[str] = []
|
||||
for ip in doh_ips:
|
||||
if ip not in seen and ip not in system_ips:
|
||||
seen.add(ip)
|
||||
candidates.append(ip)
|
||||
|
||||
# Validate through existing normalization
|
||||
validated = _normalize_fallback_ips(candidates)
|
||||
|
||||
if validated:
|
||||
logger.debug("Discovered Telegram fallback IPs via DoH: %s", ", ".join(validated))
|
||||
return validated
|
||||
|
||||
logger.info(
|
||||
"DoH discovery yielded no new IPs (system DNS: %s); using seed fallback IPs %s",
|
||||
", ".join(system_ips) or "unknown",
|
||||
", ".join(_SEED_FALLBACK_IPS),
|
||||
)
|
||||
return list(_SEED_FALLBACK_IPS)
|
||||
|
||||
|
||||
def _rewrite_request_for_ip(request: httpx.Request, ip: str) -> httpx.Request:
|
||||
original_host = request.url.host or _TELEGRAM_API_HOST
|
||||
url = request.url.copy_with(host=ip)
|
||||
headers = request.headers.copy()
|
||||
headers["host"] = original_host
|
||||
extensions = dict(request.extensions)
|
||||
extensions["sni_hostname"] = original_host
|
||||
return httpx.Request(
|
||||
method=request.method,
|
||||
url=url,
|
||||
headers=headers,
|
||||
stream=request.stream,
|
||||
extensions=extensions,
|
||||
)
|
||||
|
||||
|
||||
def _is_retryable_connect_error(exc: Exception) -> bool:
|
||||
return isinstance(exc, (httpx.ConnectTimeout, httpx.ConnectError))
|
||||
Reference in New Issue
Block a user