Fetch URL as Markdown

1"""2Fetch URL as Markdown — array form (v2.0.0).34Accepts an array of URLs and returns three parallel arrays (markdown, title,5error) plus success_count and failed_count. Wire directly from Web Search's6urls output.78Success/failure is encoded by emptiness: markdown[i] is non-empty on success9(decorated with a Source/title header) and empty on failure. error[i] carries10the reason when failed and is empty when succeeded. success_count and11failed_count are pre-computed aggregates over markdown emptiness for fast12downstream branching without iterating the arrays.1314Per-item soft-fail invariant: any single fetch that raises an SDK exception15(BackendTimeout, BackendError, RemoteError) OR returns success:false from the16sidecar is surfaced as data in the per-item slot. The node exits 0 unless17input validation fails (programmer error, exit 1).1819Concurrency: up to 4 URLs in flight via ThreadPoolExecutor.map, which preserves20input order so results[i] always corresponds to urls[i]. gais-web caps at 421process-wide as well, so client-side concurrency above 4 would just queue.2223Markdown decoration: on a successful fetch the per-item markdown is prefixed24with a single header line carrying the source URL and the extracted title.25This makes each item self-describing so downstream LLM consumers see the26provenance inline without having to cross-reference the parallel arrays. The27prefix is only added when the body is non-empty (failed fetches stay empty so28`not markdown[i]` remains a valid quick-failure check).2930No truncation: the full markdown body from gais-web flows through. The31sidecar still applies its own 1 MB cap upstream, but the signal is not32surfaced here — the template's contract is "full text or empty (failed)".33Worst-case stdout payload at N=50 can therefore exceed Temporal's 2 MB gRPC34default; this is a deliberate trade-off for faithful content delivery.35"""3637from __future__ import annotations3839import json40import sys41import traceback42from concurrent.futures import ThreadPoolExecutor43from typing import Any4445from gais import Gais464748MAX_URLS = 5049MAX_WORKERS = 4505152def parse_urls(value: Any) -> list[str]:53    """Lenient parse: accept list, or JSON-encoded string of a list.5455    Mirrors the parse_json helper in odoo-rpc — the established convention56    for Json-typed inputs in this codebase. Rejects anything that is not a57    list or a JSON string that decodes to a list.58    """59    if value is None:60        raise ValueError("Required input 'urls' not provided")61    if isinstance(value, list):62        return value63    if isinstance(value, str):64        stripped = value.strip()65        if not stripped:66            raise ValueError("Required input 'urls' is empty")67        try:68            decoded = json.loads(stripped)69        except json.JSONDecodeError as exc:70            raise ValueError(f"urls is a string but not valid JSON: {exc}") from exc71        if not isinstance(decoded, list):72            raise ValueError(73                f"urls (string) decoded to {type(decoded).__name__}, expected list"74            )75        return decoded76    raise ValueError(77        f"urls must be a list or a JSON string of a list, got {type(value).__name__}"78    )798081def validate_urls(urls: list[Any]) -> list[str]:82    """Reject empty, over-cap, or non-string inputs with a clear error."""83    if not urls:84        raise ValueError("urls cannot be empty")85    if len(urls) > MAX_URLS:86        raise ValueError(87            f"urls exceeds max {MAX_URLS} (got {len(urls)}). "88            f"Cap web-search's num_results upstream or split the batch."89        )90    for idx, item in enumerate(urls):91        if not isinstance(item, str) or not item.strip():92            raise ValueError(93                f"urls[{idx}] must be a non-empty string, got {type(item).__name__}"94            )95    return urls969798_MD_TITLE_ESCAPE = str.maketrans(99    {100        "\\": "\\\\",101        "*": "\\*",102        "_": "\\_",103        "`": "\\`",104        "[": "\\[",105        "]": "\\]",106        "(": "\\(",107        ")": "\\)",108        "\n": " ",109        "\r": " ",110    }111)112113114def decorate_markdown(url: str, title: str, markdown: str) -> str:115    """Prepend a Source/title header to the markdown body.116117    The prefix is a single bold-title + link-to-source line followed by a blank118    line, so it composes cleanly with whatever heading the page itself emits119    below. Title can be empty (sidecar best-effort) — fall back to URL-only.120121    Title is escaped so a hostile <title> tag (containing `**`, `]`, newlines,122    etc.) cannot break the markdown parse or collide with the chat card's123    `[Source](url)` extractor regex.124125    Returns the original body unchanged when it is empty (a failed fetch126    should NOT gain a misleading provenance header).127    """128    if not markdown:129        return markdown130    safe_title = (title or "").strip().translate(_MD_TITLE_ESCAPE)131    # Collapse runs of whitespace introduced by newline-stripping.132    safe_title = " ".join(safe_title.split())133    if safe_title:134        header = f"**{safe_title}** — [Source]({url})"135    else:136        header = f"[Source]({url})"137    return f"{header}\n\n{markdown}"138139140def fetch_one(url: str, timeout_seconds: int, idx: int, total: int) -> dict[str, Any]:141    """Fetch one URL. NEVER raises — exceptions become soft-fail data.142143    Returns 3 fields per item: markdown (empty on failure), title, error.144    Success/failure is encoded by markdown emptiness; the explicit success145    bool was removed as redundant.146    """147    try:148        print(149            f"[fetch-url-markdown][{idx + 1}/{total}] start url={url!r} "150            f"timeout={timeout_seconds}s",151            file=sys.stderr,152        )153        result = Gais.web.fetch(url=url, timeout_seconds=timeout_seconds)154        md = result.metadata155        markdown = md.get("markdown", "") or ""156        title = md.get("title", "") or ""157        success = bool(md.get("success", False))158        error = md.get("error", "") or ""159160        # Soft-fail unification: a sidecar success=False with an empty body161        # MUST collapse markdown to "" so `not markdown[i]` is the single162        # source of truth for "this item failed". A sidecar success=False163        # with a non-empty body (rare) is treated as failed too.164        if not success:165            markdown = ""166167        # Decorate non-empty successful bodies with provenance.168        if markdown:169            markdown = decorate_markdown(url, title, markdown)170171        print(172            f"[fetch-url-markdown][{idx + 1}/{total}] done "173            f"markdown_chars={len(markdown)} error={error!r}",174            file=sys.stderr,175        )176        return {177            "markdown": markdown,178            "title": title,179            "error": error,180        }181    except Exception as exc:  # noqa: BLE001 — soft-fail by contract182        err_label = f"{type(exc).__name__}: {exc}"183        print(184            f"[fetch-url-markdown][{idx + 1}/{total}] FAILED {err_label}",185            file=sys.stderr,186        )187        return {188            "markdown": "",189            "title": "",190            "error": err_label,191        }192193194def process(inputs: dict[str, Any]) -> dict[str, Any]:195    """Run the batch fetch and return the 5-field output dict.196197    Pure function over the input dict. Raises ValueError on input validation198    failures (programmer error); per-item SDK exceptions are caught inside199    fetch_one and surfaced as soft-fail data (empty markdown + populated200    error). Extracted from main() so tests can drive it without stdin/stdout201    monkey-patching.202    """203    urls = validate_urls(parse_urls(inputs.get("urls")))204    timeout_seconds = int(inputs.get("timeout_seconds") or 30)205    if not (5 <= timeout_seconds <= 120):206        raise ValueError(207            f"timeout_seconds must be between 5 and 120, got {timeout_seconds}"208        )209210    total = len(urls)211    print(212        f"[fetch-url-markdown] batch start n={total} "213        f"timeout={timeout_seconds}s workers={MAX_WORKERS}",214        file=sys.stderr,215    )216217    # ThreadPoolExecutor.map preserves order: results[i] <-> urls[i].218    # gais-web's process-wide semaphore caps concurrency at 4 server-side,219    # so MAX_WORKERS=4 client-side aligns perfectly.220    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:221        results = list(222            pool.map(223                lambda iu: fetch_one(iu[1], timeout_seconds, iu[0], total),224                enumerate(urls),225            )226        )227228    # Pivot: list-of-dicts -> 3 parallel arrays. Order preserved by pool.map.229    markdown_arr = [r["markdown"] for r in results]230    title_arr = [r["title"] for r in results]231    error_arr = [r["error"] for r in results]232233    # Aggregates derived from markdown emptiness (the single failure signal234    # after dropping the redundant success[] and the no-longer-meaningful235    # truncated[] arrays).236    success_count = sum(1 for m in markdown_arr if m)237    failed_count = total - success_count238239    print(240        f"[fetch-url-markdown] batch done n={total} success={success_count} "241        f"failed={failed_count}",242        file=sys.stderr,243    )244245    return {246        "markdown": markdown_arr,247        "title": title_arr,248        "error": error_arr,249        "success_count": success_count,250        "failed_count": failed_count,251    }252253254def main() -> None:255    try:256        envelope = json.loads(sys.stdin.read() or "{}")257        inputs: dict[str, Any] = (258            envelope.get("inputs", {}) if isinstance(envelope, dict) else {}259        )260        output = process(inputs)261        json.dump(output, sys.stdout)262    except Exception as e:263        error_payload = {264            "error": str(e),265            "errorType": type(e).__name__,266            "traceback": traceback.format_exc(),267        }268        print(json.dumps(error_payload), file=sys.stderr)269        sys.exit(1)270271272if __name__ == "__main__":273    main()