AzurLaneAutoScript/module/smart_mgmt/fetcher_test.py

from __future__ import annotations

import json
import socket
import ssl
import time
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from pathlib import Path
from urllib.parse import urlparse

import requests
from rich.table import Table
from rich.text import Text

from module.logger import logger
from module.smart_mgmt.downtime_fetcher import DowntimeFetcher


class _TestStatus(Enum):
    PASS = 'PASS'
    FAIL = 'FAIL'
    SKIP = 'SKIP'


@dataclass
class _TestResult:
    name: str
    status: _TestStatus
    note: str


class DowntimeFetcherTest:
    # TCP/TLS probe timeout, shorter than HTTP request timeout for faster fail
    CONNECT_TIMEOUT = 10.0

    # Number of articles (excluding the latest) shown as title-only preview
    ARTICLE_PREVIEW_COUNT = 3
    # Max summary length shown for the latest article
    SUMMARY_EXCERPT_LEN = 200
    ARTICLES_JSONL_FILE = './log/downtime_articles_cache.jsonl'

    # Test names
    TEST_NETWORK = 'Network'
    TEST_LISTS_API = 'Lists API'
    TEST_COLLECTION = 'Collection lookup'
    TEST_ARTICLES_API = 'Articles API'
    TEST_PARSE = 'Parse window'

    _STATUS_STYLE = {
        _TestStatus.PASS: 'green',
        _TestStatus.FAIL: 'bright_red',
        _TestStatus.SKIP: 'yellow',
    }

    def __init__(self):
        self.fetcher = DowntimeFetcher()
        # Abstract main-flow params into local test params: single point of
        # change when fetcher internals evolve, avoids scattered coupling.
        self.session = self.fetcher.session
        self.mid = self.fetcher.DEFAULT_MID
        self.collection_name = self.fetcher.DEFAULT_COLLECTION_NAME
        self.lists_api = self.fetcher.ARTICLE_LISTS_API
        self.articles_api = self.fetcher.ARTICLE_COLLECTION_API
        self.request_timeout = self.fetcher.REQUEST_TIMEOUT
        self.cn_tz = self.fetcher.CN_TZ
        self.window_re = self.fetcher.WINDOW_RE
        self.duration_re = self.fetcher.DURATION_RE
        self.title_time_re = self.fetcher.TITLE_TIME_RE
        # State carried across tests along the dependency chain
        self._lists = []
        self._up_name = ''
        self._collection_id = None
        self._latest_article = None
        self._results = []
        self._skip_remaining = False

    def run(self):
        logger.hr('Downtime fetcher test', level=1)
        self._results.clear()
        self._skip_remaining = False

        tests = [
            (self.TEST_NETWORK, self.test_network),
            (self.TEST_LISTS_API, self.test_lists_api),
            (self.TEST_COLLECTION, self.test_find_collection),
            (self.TEST_ARTICLES_API, self.test_articles_api),
            (self.TEST_PARSE, self.test_parse_window),
        ]
        for name, func in tests:
            if self._skip_remaining:
                self._results.append(_TestResult(name, _TestStatus.SKIP, 'Upstream failed'))
                continue
            try:
                func()
            except Exception as e:
                # Unexpected programming error: record and stop the chain so
                # the summary still prints
                logger.exception(f'Unexpected error in {name}')
                self._results.append(_TestResult(name, _TestStatus.FAIL, f'unexpected: {type(e).__name__}: {e}'))
                self._skip_remaining = True
                continue
            if self._results[-1].status == _TestStatus.FAIL:
                self._skip_remaining = True

        self._show_summary()

    # ------------------------------------------------------------------
    # Shared helpers
    # ------------------------------------------------------------------

    def _fetch_json(self, url, params):
        """GET url with fetcher session and return (body, error).

        Returns:
            tuple: (parsed_body, None) on success; (None, diagnostic_str) on failure.
        """
        logger.info(f'Request: {url} params={params}')
        try:
            resp = self.session.get(url, params=params, timeout=self.request_timeout)
            logger.info(f'HTTP status: {resp.status_code}')
            return resp.json(), None
        except requests.RequestException as e:
            logger.warning(f'Request failed: {e}')
            return None, f'{type(e).__name__}: {e}'
        except ValueError as e:
            logger.warning(f'JSON parse failed: {e}')
            return None, f'JSON: {e}'

    def _log_body_structure(self, body):
        """Log top-level keys and data keys of API response body."""
        if not isinstance(body, dict):
            logger.info(f'Body is {type(body).__name__}, not dict')
            return
        logger.info(f'Body top-level keys: {list(body.keys())}')
        data = body.get('data')
        if isinstance(data, dict):
            logger.info(f'data keys: {list(data.keys())}')
        elif data is not None:
            logger.info(f'data is {type(data).__name__}, not dict')

    def _format_pub_time(self, pub_ts):
        """Format publish timestamp to readable CN time string."""
        return datetime.fromtimestamp(pub_ts, tz=self.cn_tz).strftime('%Y-%m-%d %H:%M')

    def _normalize_article_for_cache(self, article):
        """Convert raw API article dict to cache record format.

        Mirrors dev_tools/downtime_notice_crawler.py normalize_article output
        so test caches stay consistent with crawler-produced caches.
        """
        cvid = str(article['id'])
        pub_ts = int(article['publish_time'])
        return {
            'pub_ts': pub_ts,
            'pub_time': datetime.fromtimestamp(pub_ts, tz=self.cn_tz).isoformat(),
            'title': article['title'],
            'text': article['summary'],
            'url': f'https://www.bilibili.com/read/cv{cvid}',
            'collection_id': self._collection_id,
            'collection_name': self.collection_name,
        }

    def _save_caches(self, articles):
        """Persist normalized article records for diagnostic inspection.

        Args:
            articles (list[dict]): Raw article list from body['data']['articles'].
        """
        Path(self.ARTICLES_JSONL_FILE).parent.mkdir(parents=True, exist_ok=True)
        records = [self._normalize_article_for_cache(a) for a in articles]
        jsonl_path = Path(self.ARTICLES_JSONL_FILE)
        jsonl_path.write_text(
            ''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in records),
            encoding='utf-8',
        )
        logger.info(f'Saved {len(records)} article records to {self.ARTICLES_JSONL_FILE}')

    def _record(self, name, status, note=''):
        self._results.append(_TestResult(name, status, note))

    # ------------------------------------------------------------------
    # Test cases
    # ------------------------------------------------------------------

    def test_network(self):
        """Probe DNS/TCP/TLS to the API host to locate where connectivity breaks."""
        logger.hr('Network connectivity', level=2)
        host = urlparse(self.lists_api).hostname
        port = 443
        logger.info(f'Target: {host}:{port}')

        try:
            t0 = time.monotonic()
            ip = socket.gethostbyname(host)
            dns_cost = time.monotonic() - t0
            logger.info(f'DNS resolved: {host} -> {ip} ({dns_cost:.2f}s)')
        except socket.gaierror as e:
            logger.warning(f'DNS resolution failed: {e}')
            self._record(self.TEST_NETWORK, _TestStatus.FAIL, f'DNS: {e}')
            return

        ctx = ssl.create_default_context()
        sock = None
        try:
            t0 = time.monotonic()
            sock = socket.create_connection((host, port), timeout=self.CONNECT_TIMEOUT)
            tcp_cost = time.monotonic() - t0
            logger.info(f'TCP connected: {host}:{port} ({tcp_cost:.2f}s)')

            t1 = time.monotonic()
            ssock = ctx.wrap_socket(sock, server_hostname=host)
            tls_cost = time.monotonic() - t1
            ssock.close()
            sock = None
            logger.info(f'TLS handshake ok ({tls_cost:.2f}s)')
            self._record(
                self.TEST_NETWORK, _TestStatus.PASS,
                f'DNS:{dns_cost:.2f}s TCP:{tcp_cost:.2f}s TLS:{tls_cost:.2f}s',
            )
        except OSError as e:
            logger.warning(f'Connection failed: {type(e).__name__}: {e}')
            self._record(self.TEST_NETWORK, _TestStatus.FAIL, f'{type(e).__name__}: {e}')
        finally:
            if sock is not None:
                try:
                    sock.close()
                except OSError:
                    pass

    def test_lists_api(self):
        """Verify ARTICLE_LISTS_API returns valid response with data.lists."""
        logger.hr('Lists API', level=2)
        body, err = self._fetch_json(self.lists_api, {'mid': str(self.mid), 'sort': 0})
        if body is None:
            self._record(self.TEST_LISTS_API, _TestStatus.FAIL, err)
            return

        self._log_body_structure(body)
        code = body.get('code')
        message = body.get('message') or body.get('msg') or ''
        logger.info(f'API response: code={code} message={message!r}')
        if code != 0:
            logger.warning(f'Non-zero code, full body: {body}')
            self._record(self.TEST_LISTS_API, _TestStatus.FAIL, f'code={code} msg={message}')
            return

        lists = (body.get('data') or {}).get('lists')
        if not isinstance(lists, list):
            logger.warning(f'Invalid data.lists: type={type(lists).__name__}')
            self._record(self.TEST_LISTS_API, _TestStatus.FAIL, f'lists invalid: {type(lists).__name__}')
            return

        self._lists = lists
        logger.info(f'Got {len(lists)} collections')
        self._record(self.TEST_LISTS_API, _TestStatus.PASS, f'{len(lists)} collections')

    def test_find_collection(self):
        """Check whether target collection exists in UP's lists."""
        logger.hr('Collection lookup', level=2)
        logger.info(f'Target collection: {self.collection_name!r}')
        names = [item.get('name') for item in self._lists]
        logger.info(f'Available collections: {names}')

        target = next(
            (item for item in self._lists if item.get('name') == self.collection_name),
            None,
        )
        if target is None:
            logger.warning(f'Target not found: {self.collection_name!r}')
            self._record(self.TEST_COLLECTION, _TestStatus.FAIL, f'not found, available={names}')
            return

        self._collection_id = int(target['id'])
        logger.info(f'Found: id={self._collection_id} name={target["name"]!r}')
        self._record(self.TEST_COLLECTION, _TestStatus.PASS, f'id={self._collection_id}')

    def test_articles_api(self):
        """Verify ARTICLE_COLLECTION_API returns articles for the collection."""
        logger.hr('Articles API', level=2)
        body, err = self._fetch_json(self.articles_api, {'id': str(self._collection_id)})
        if body is None:
            self._record(self.TEST_ARTICLES_API, _TestStatus.FAIL, err)
            return

        self._log_body_structure(body)
        code = body.get('code')
        message = body.get('message') or body.get('msg') or ''
        logger.info(f'API response: code={code} message={message!r}')
        if code != 0:
            logger.warning(f'Non-zero code, full body: {body}')
            self._record(self.TEST_ARTICLES_API, _TestStatus.FAIL, f'code={code} msg={message}')
            return

        articles = (body.get('data') or {}).get('articles')
        if not isinstance(articles, list) or not articles:
            logger.warning(f'Empty or invalid data.articles: type={type(articles).__name__}')
            self._record(self.TEST_ARTICLES_API, _TestStatus.FAIL, f'articles empty: {type(articles).__name__}')
            return

        # Extract UP name from response body for display
        author = (body.get('data') or {}).get('author')
        if isinstance(author, dict):
            up_name = author.get('name')
            if up_name:
                self._up_name = up_name
                logger.info(f'UP: mid={self.mid} name={up_name!r}')

        logger.info(f'The latest article and recent {self.ARTICLE_PREVIEW_COUNT} articles:')

        # Latest article: title + summary excerpt
        sorted_articles = sorted(articles, key=lambda a: a.get('publish_time', 0), reverse=True)
        latest = sorted_articles[0]
        pub_time = self._format_pub_time(latest.get('publish_time', 0))
        summary = (latest.get('summary') or '')[:self.SUMMARY_EXCERPT_LEN]
        logger.info(f'Latest: [{pub_time}] {latest.get("title")} | {summary!r}')

        # Next N articles: title only
        for a in sorted_articles[1:1 + self.ARTICLE_PREVIEW_COUNT]:
            pub_time = self._format_pub_time(a.get('publish_time', 0))
            logger.info(f'[{pub_time}] {a.get("title")}')

        # Persist article records for diagnostic inspection
        self._save_caches(articles)

        articles_normalized = [self.fetcher._normalize_article(a) for a in articles]
        self._latest_article = self.fetcher._latest_article(articles_normalized)
        self._record(self.TEST_ARTICLES_API, _TestStatus.PASS, f'{len(articles)} articles')

    def test_parse_window(self):
        """Run regex parse against the latest article to detect format drift."""
        logger.hr('Parse window', level=2)
        article = self._latest_article
        body = self.fetcher._clean_text(article.summary)

        # Show individual regex match results for diagnosis
        window_match = self.window_re.search(body)
        duration_match = self.duration_re.search(body)
        title_match = self.title_time_re.search(article.title)
        logger.info(f'WINDOW_RE on summary: {"matched" if window_match else "unmatched"} '
                    f'groups={window_match.groups() if window_match else None}')
        logger.info(f'DURATION_RE on summary: {"matched" if duration_match else "unmatched"} '
                    f'groups={duration_match.groups() if duration_match else None}')
        logger.info(f'TITLE_TIME_RE on title: {"matched" if title_match else "unmatched"} '
                    f'groups={title_match.groups() if title_match else None}')

        window = self.fetcher.parse_window_from_article(article)
        if window is None:
            logger.warning('parse_window_from_article returned None')
            self._record(self.TEST_PARSE, _TestStatus.FAIL, 'no regex matched')
            return

        logger.info(f'Parsed window: {window.window!r}')
        self._record(self.TEST_PARSE, _TestStatus.PASS, f'window={window.window}')

    # ------------------------------------------------------------------
    # Summary
    # ------------------------------------------------------------------

    def _show_summary(self):
        logger.hr('Test summary', level=1)
        table = Table(show_lines=True)
        table.add_column('Test', style='cyan', no_wrap=True)
        table.add_column('Result')
        table.add_column('Note')
        for r in self._results:
            style = self._STATUS_STYLE.get(r.status, 'white')
            table.add_row(r.name, Text(r.status.value, style=style), r.note)
        logger.print(table, justify='center')