from __future__ import annotations import json import socket import ssl import time from dataclasses import dataclass from datetime import datetime from enum import Enum from pathlib import Path from urllib.parse import urlparse import requests from rich.table import Table from rich.text import Text from module.logger import logger from module.smart_mgmt.downtime_fetcher import DowntimeFetcher class _TestStatus(Enum): PASS = 'PASS' FAIL = 'FAIL' SKIP = 'SKIP' @dataclass class _TestResult: name: str status: _TestStatus note: str class DowntimeFetcherTest: # TCP/TLS probe timeout, shorter than HTTP request timeout for faster fail CONNECT_TIMEOUT = 10.0 # Number of articles (excluding the latest) shown as title-only preview ARTICLE_PREVIEW_COUNT = 3 # Max summary length shown for the latest article SUMMARY_EXCERPT_LEN = 200 ARTICLES_JSONL_FILE = './log/downtime_articles_cache.jsonl' # Test names TEST_NETWORK = 'Network' TEST_LISTS_API = 'Lists API' TEST_COLLECTION = 'Collection lookup' TEST_ARTICLES_API = 'Articles API' TEST_PARSE = 'Parse window' _STATUS_STYLE = { _TestStatus.PASS: 'green', _TestStatus.FAIL: 'bright_red', _TestStatus.SKIP: 'yellow', } def __init__(self): self.fetcher = DowntimeFetcher() # Abstract main-flow params into local test params: single point of # change when fetcher internals evolve, avoids scattered coupling. self.session = self.fetcher.session self.mid = self.fetcher.DEFAULT_MID self.collection_name = self.fetcher.DEFAULT_COLLECTION_NAME self.lists_api = self.fetcher.ARTICLE_LISTS_API self.articles_api = self.fetcher.ARTICLE_COLLECTION_API self.request_timeout = self.fetcher.REQUEST_TIMEOUT self.cn_tz = self.fetcher.CN_TZ self.window_re = self.fetcher.WINDOW_RE self.duration_re = self.fetcher.DURATION_RE self.title_time_re = self.fetcher.TITLE_TIME_RE # State carried across tests along the dependency chain self._lists = [] self._up_name = '' self._collection_id = None self._latest_article = None self._results = [] self._skip_remaining = False def run(self): logger.hr('Downtime fetcher test', level=1) self._results.clear() self._skip_remaining = False tests = [ (self.TEST_NETWORK, self.test_network), (self.TEST_LISTS_API, self.test_lists_api), (self.TEST_COLLECTION, self.test_find_collection), (self.TEST_ARTICLES_API, self.test_articles_api), (self.TEST_PARSE, self.test_parse_window), ] for name, func in tests: if self._skip_remaining: self._results.append(_TestResult(name, _TestStatus.SKIP, 'Upstream failed')) continue try: func() except Exception as e: # Unexpected programming error: record and stop the chain so # the summary still prints logger.exception(f'Unexpected error in {name}') self._results.append(_TestResult(name, _TestStatus.FAIL, f'unexpected: {type(e).__name__}: {e}')) self._skip_remaining = True continue if self._results[-1].status == _TestStatus.FAIL: self._skip_remaining = True self._show_summary() # ------------------------------------------------------------------ # Shared helpers # ------------------------------------------------------------------ def _fetch_json(self, url, params): """GET url with fetcher session and return (body, error). Returns: tuple: (parsed_body, None) on success; (None, diagnostic_str) on failure. """ logger.info(f'Request: {url} params={params}') try: resp = self.session.get(url, params=params, timeout=self.request_timeout) logger.info(f'HTTP status: {resp.status_code}') return resp.json(), None except requests.RequestException as e: logger.warning(f'Request failed: {e}') return None, f'{type(e).__name__}: {e}' except ValueError as e: logger.warning(f'JSON parse failed: {e}') return None, f'JSON: {e}' def _log_body_structure(self, body): """Log top-level keys and data keys of API response body.""" if not isinstance(body, dict): logger.info(f'Body is {type(body).__name__}, not dict') return logger.info(f'Body top-level keys: {list(body.keys())}') data = body.get('data') if isinstance(data, dict): logger.info(f'data keys: {list(data.keys())}') elif data is not None: logger.info(f'data is {type(data).__name__}, not dict') def _format_pub_time(self, pub_ts): """Format publish timestamp to readable CN time string.""" return datetime.fromtimestamp(pub_ts, tz=self.cn_tz).strftime('%Y-%m-%d %H:%M') def _normalize_article_for_cache(self, article): """Convert raw API article dict to cache record format. Mirrors dev_tools/downtime_notice_crawler.py normalize_article output so test caches stay consistent with crawler-produced caches. """ cvid = str(article['id']) pub_ts = int(article['publish_time']) return { 'pub_ts': pub_ts, 'pub_time': datetime.fromtimestamp(pub_ts, tz=self.cn_tz).isoformat(), 'title': article['title'], 'text': article['summary'], 'url': f'https://www.bilibili.com/read/cv{cvid}', 'collection_id': self._collection_id, 'collection_name': self.collection_name, } def _save_caches(self, articles): """Persist normalized article records for diagnostic inspection. Args: articles (list[dict]): Raw article list from body['data']['articles']. """ Path(self.ARTICLES_JSONL_FILE).parent.mkdir(parents=True, exist_ok=True) records = [self._normalize_article_for_cache(a) for a in articles] jsonl_path = Path(self.ARTICLES_JSONL_FILE) jsonl_path.write_text( ''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in records), encoding='utf-8', ) logger.info(f'Saved {len(records)} article records to {self.ARTICLES_JSONL_FILE}') def _record(self, name, status, note=''): self._results.append(_TestResult(name, status, note)) # ------------------------------------------------------------------ # Test cases # ------------------------------------------------------------------ def test_network(self): """Probe DNS/TCP/TLS to the API host to locate where connectivity breaks.""" logger.hr('Network connectivity', level=2) host = urlparse(self.lists_api).hostname port = 443 logger.info(f'Target: {host}:{port}') try: t0 = time.monotonic() ip = socket.gethostbyname(host) dns_cost = time.monotonic() - t0 logger.info(f'DNS resolved: {host} -> {ip} ({dns_cost:.2f}s)') except socket.gaierror as e: logger.warning(f'DNS resolution failed: {e}') self._record(self.TEST_NETWORK, _TestStatus.FAIL, f'DNS: {e}') return ctx = ssl.create_default_context() sock = None try: t0 = time.monotonic() sock = socket.create_connection((host, port), timeout=self.CONNECT_TIMEOUT) tcp_cost = time.monotonic() - t0 logger.info(f'TCP connected: {host}:{port} ({tcp_cost:.2f}s)') t1 = time.monotonic() ssock = ctx.wrap_socket(sock, server_hostname=host) tls_cost = time.monotonic() - t1 ssock.close() sock = None logger.info(f'TLS handshake ok ({tls_cost:.2f}s)') self._record( self.TEST_NETWORK, _TestStatus.PASS, f'DNS:{dns_cost:.2f}s TCP:{tcp_cost:.2f}s TLS:{tls_cost:.2f}s', ) except OSError as e: logger.warning(f'Connection failed: {type(e).__name__}: {e}') self._record(self.TEST_NETWORK, _TestStatus.FAIL, f'{type(e).__name__}: {e}') finally: if sock is not None: try: sock.close() except OSError: pass def test_lists_api(self): """Verify ARTICLE_LISTS_API returns valid response with data.lists.""" logger.hr('Lists API', level=2) body, err = self._fetch_json(self.lists_api, {'mid': str(self.mid), 'sort': 0}) if body is None: self._record(self.TEST_LISTS_API, _TestStatus.FAIL, err) return self._log_body_structure(body) code = body.get('code') message = body.get('message') or body.get('msg') or '' logger.info(f'API response: code={code} message={message!r}') if code != 0: logger.warning(f'Non-zero code, full body: {body}') self._record(self.TEST_LISTS_API, _TestStatus.FAIL, f'code={code} msg={message}') return lists = (body.get('data') or {}).get('lists') if not isinstance(lists, list): logger.warning(f'Invalid data.lists: type={type(lists).__name__}') self._record(self.TEST_LISTS_API, _TestStatus.FAIL, f'lists invalid: {type(lists).__name__}') return self._lists = lists logger.info(f'Got {len(lists)} collections') self._record(self.TEST_LISTS_API, _TestStatus.PASS, f'{len(lists)} collections') def test_find_collection(self): """Check whether target collection exists in UP's lists.""" logger.hr('Collection lookup', level=2) logger.info(f'Target collection: {self.collection_name!r}') names = [item.get('name') for item in self._lists] logger.info(f'Available collections: {names}') target = next( (item for item in self._lists if item.get('name') == self.collection_name), None, ) if target is None: logger.warning(f'Target not found: {self.collection_name!r}') self._record(self.TEST_COLLECTION, _TestStatus.FAIL, f'not found, available={names}') return self._collection_id = int(target['id']) logger.info(f'Found: id={self._collection_id} name={target["name"]!r}') self._record(self.TEST_COLLECTION, _TestStatus.PASS, f'id={self._collection_id}') def test_articles_api(self): """Verify ARTICLE_COLLECTION_API returns articles for the collection.""" logger.hr('Articles API', level=2) body, err = self._fetch_json(self.articles_api, {'id': str(self._collection_id)}) if body is None: self._record(self.TEST_ARTICLES_API, _TestStatus.FAIL, err) return self._log_body_structure(body) code = body.get('code') message = body.get('message') or body.get('msg') or '' logger.info(f'API response: code={code} message={message!r}') if code != 0: logger.warning(f'Non-zero code, full body: {body}') self._record(self.TEST_ARTICLES_API, _TestStatus.FAIL, f'code={code} msg={message}') return articles = (body.get('data') or {}).get('articles') if not isinstance(articles, list) or not articles: logger.warning(f'Empty or invalid data.articles: type={type(articles).__name__}') self._record(self.TEST_ARTICLES_API, _TestStatus.FAIL, f'articles empty: {type(articles).__name__}') return # Extract UP name from response body for display author = (body.get('data') or {}).get('author') if isinstance(author, dict): up_name = author.get('name') if up_name: self._up_name = up_name logger.info(f'UP: mid={self.mid} name={up_name!r}') logger.info(f'The latest article and recent {self.ARTICLE_PREVIEW_COUNT} articles:') # Latest article: title + summary excerpt sorted_articles = sorted(articles, key=lambda a: a.get('publish_time', 0), reverse=True) latest = sorted_articles[0] pub_time = self._format_pub_time(latest.get('publish_time', 0)) summary = (latest.get('summary') or '')[:self.SUMMARY_EXCERPT_LEN] logger.info(f'Latest: [{pub_time}] {latest.get("title")} | {summary!r}') # Next N articles: title only for a in sorted_articles[1:1 + self.ARTICLE_PREVIEW_COUNT]: pub_time = self._format_pub_time(a.get('publish_time', 0)) logger.info(f'[{pub_time}] {a.get("title")}') # Persist article records for diagnostic inspection self._save_caches(articles) articles_normalized = [self.fetcher._normalize_article(a) for a in articles] self._latest_article = self.fetcher._latest_article(articles_normalized) self._record(self.TEST_ARTICLES_API, _TestStatus.PASS, f'{len(articles)} articles') def test_parse_window(self): """Run regex parse against the latest article to detect format drift.""" logger.hr('Parse window', level=2) article = self._latest_article body = self.fetcher._clean_text(article.summary) # Show individual regex match results for diagnosis window_match = self.window_re.search(body) duration_match = self.duration_re.search(body) title_match = self.title_time_re.search(article.title) logger.info(f'WINDOW_RE on summary: {"matched" if window_match else "unmatched"} ' f'groups={window_match.groups() if window_match else None}') logger.info(f'DURATION_RE on summary: {"matched" if duration_match else "unmatched"} ' f'groups={duration_match.groups() if duration_match else None}') logger.info(f'TITLE_TIME_RE on title: {"matched" if title_match else "unmatched"} ' f'groups={title_match.groups() if title_match else None}') window = self.fetcher.parse_window_from_article(article) if window is None: logger.warning('parse_window_from_article returned None') self._record(self.TEST_PARSE, _TestStatus.FAIL, 'no regex matched') return logger.info(f'Parsed window: {window.window!r}') self._record(self.TEST_PARSE, _TestStatus.PASS, f'window={window.window}') # ------------------------------------------------------------------ # Summary # ------------------------------------------------------------------ def _show_summary(self): logger.hr('Test summary', level=1) table = Table(show_lines=True) table.add_column('Test', style='cyan', no_wrap=True) table.add_column('Result') table.add_column('Note') for r in self._results: style = self._STATUS_STYLE.get(r.status, 'white') table.add_row(r.name, Text(r.status.value, style=style), r.note) logger.print(table, justify='center')