PDF Translator

1"""pdf-translator — translate one Drive document into another language, keeping its layout.23BabelDOC does the translation, behind one gais call. This node only checks4its inputs, fetches the files from Drive, and names the results:56  1. Validate the inputs (Drive ids, languages, pages) before touching Drive.7  2. ``Gais.drive.download(item_id, as_pdf=True)``: the item's PDF. A PDF upload8     comes as it is; a Word, PowerPoint or text upload as the PDF Drive made of9     it, so every input comes back as a translated PDF (decision C1). A10     spreadsheet, image, audio or video item, or one still processing, answers11     422 and the route's own message is shown.12  3. The download must be a PDF (``%PDF-`` in its first KiB). An older13     www-emblema ignores ``as=pdf`` and returns the original file, which would14     otherwise reach BabelDOC as a confusing failure.15  4. The glossary, when given: ``Gais.drive.download(item_id)`` (the original16     bytes), CSV or XLSX, copied to a scratch file with the right extension.17  5. ``Gais.pdf_translate.translate``: BabelDOC behind the billing shim, the18     every-passage ledger and the leftover-text and number checks all live in19     that gais adapter. It raises ``PdfTranslateError`` (a failure class and a20     content-free message), or returns the translated PDF, the bilingual PDF21     when asked, and the report.22  6. Copy the PDFs to /data/output as ``<name>.<target>.pdf`` and23     ``<name>.<target>.bilingual.pdf``; add the glossary line to the report.2425No document text in errors or logs (decision C5). The worker logs a node's26whole stderr, which ships to Loki. So:27  * a ``PdfTranslateError`` prints ``{"error", "errorType"}`` and no traceback;28  * a Drive failure prints the www route's own message, never a raw body;29  * input errors describe the expected value and never echo the given one (an30    edge can carry any upstream text);31  * any other exception prints its type and the stack frames, never its32    message, since a message can quote what it failed on;33  * nothing is printed on success.3435The node itself imports no HTTP client and no os/pathlib/shutil (lint): the36token-bearing Drive calls are inside gais.37"""3839from __future__ import annotations4041import hashlib42import json43import re44import sys45import tempfile46import traceback47from typing import Any4849from gais import BackendTimeout, Gais, GaisError, InvalidParams, RemoteError5051OUTPUT_DIR = "/data/output"52#: Parent of the run's scratch directories (None = tempfile's default).53SCRATCH_DIR: str | None = None5455#: The languages the adapter translates between: the platform's own list and56#: order (www-emblema `AVAILABLE_LANGUAGES`, the settings page's language57#: select), with English names.58LANGUAGES: dict[str, str] = {59    "it": "Italian",60    "en": "English",61    "fr": "French",62    "de": "German",63    "es": "Spanish",64    "ja": "Japanese",65    "pt": "Portuguese",66    "zh": "Chinese",67    "ko": "Korean",68    "ru": "Russian",69    "nl": "Dutch",70}71#: The names the settings page shows, accepted from an edge as well.72NATIVE_NAMES: dict[str, str] = {73    "it": "Italiano", "en": "English", "fr": "Français", "de": "Deutsch", "es": "Español",74    "ja": "日本語", "pt": "Português", "zh": "中文", "ko": "한국어", "ru": "Русский",75    "nl": "Nederlands",76}77DEFAULT_TARGET = "it"78#: The adapter reads the source language from the document's text.79AUTO = "auto"80DEFAULT_SOURCE = AUTO81DEFAULT_NAME = "document"8283_UUID_RE = re.compile(84    r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",85    re.IGNORECASE,86)87_FILE_MENTION = "mention://drive_file/"88_FOLDER_MENTION = "mention://drive_folder/"8990# "it", "IT", "it-IT", "pt_BR"; full English names are accepted too.91_LANGUAGE_RE = re.compile(r"([a-z]{2})(?:[-_][a-z0-9]{2,8})?")92_LANGUAGE_BY_NAME = {93    name.lower(): code94    for names in (LANGUAGES, NATIVE_NAMES)95    for code, name in names.items()96}9798# "1-5,8": page numbers and closed ranges, after whitespace is removed.99_PAGES_RE = re.compile(r"\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*")100101# Readers accept the header anywhere in the first KiB.102_PDF_MAGIC = b"%PDF-"103_PDF_HEADER_WINDOW = 1024104105# Output names follow the PDF Builder's rules: path and Windows-reserved106# characters, `%` (keys are URL-decoded before their traversal check), `#` and107# `?`, controls. The byte cap keeps `<stem>.<lang>.bilingual.pdf` under the108# 255-byte file-name limit for names in non-Latin scripts.109_UNSAFE_NAME_CHARS = re.compile(r'[\x00-\x1f\x7f<>:"/\\|?*%#]')110_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]")111_MAX_NAME_CHARS = 120112_MAX_NAME_BYTES = 200113# The source file's own extension, dropped from the output name.114_SOURCE_EXTENSION = re.compile(115    r"\.(?:pdf|docx?|pptx?|odt|odp|rtf|txt|md|markdown|html?)$", re.IGNORECASE116)117118_GLOSSARY_KINDS = ("csv", "xlsx")119_GLOSSARY_MIME_KINDS = {120    "text/csv": "csv",121    "application/csv": "csv",122    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",123}124125_ROUTE_CODE_RE = re.compile(r"[A-Z][A-Z0-9_]{1,63}")126_MAX_MESSAGE_CHARS = 300127_COPY_CHUNK = 1024 * 1024128129_REBUILD_EXECUTOR = (130    "The PDF Translator is not installed in this server's Python executor image: "131    "an operator must rebuild emblema/workspace-executor-python"132)133_NOT_A_PDF = (134    "Drive did not return a PDF for this document. This server's www-emblema may "135    "predate the PDF Translator (an operator must update it), or the stored PDF is damaged"136)137_NO_PDF = (138    "This Drive file has no PDF to translate, or it is still being processed. "139    "Spreadsheets, images, audio and video cannot be translated here; "140    "a new upload can be translated once its processing ends"141)142_PAGES_FORMAT = (143    "Pages must be page numbers and ranges separated by commas, counting from 1, "144    "e.g. 1-5,8"145)146147148class NodeError(Exception):149    """A failure whose message is safe to show and to log.150151    It says what went wrong and what to do, never document text (C5).152    """153154    def __init__(self, message: str, error_type: str) -> None:155        super().__init__(message)156        self.error_type = error_type157158159# ── inputs ──────────────────────────────────────────────────────────────────160161162def _drive_item_id(value: Any, label: str) -> str | None:163    """A bare UUID or ``mention://drive_file/<uuid>``; None when not given.164165    A one-element list is accepted (the picker may store one).166    """167    if isinstance(value, list):168        items = [v for v in value if v not in (None, "")]169        if len(items) > 1:170            raise NodeError(171                f"{label} takes one Drive file, got {len(items)}; "172                "use one PDF Translator per file",173                "invalid_input",174            )175        value = items[0] if items else None176    if value is None:177        return None178    if not isinstance(value, str):179        raise NodeError(180            f"{label} must be a Drive file id, got a {type(value).__name__}", "invalid_input"181        )182    text = value.strip()183    if not text:184        return None185    if text.startswith(_FOLDER_MENTION):186        raise NodeError(f"{label} is a Drive folder; pick a file", "invalid_input")187    if text.startswith(_FILE_MENTION):188        text = text[len(_FILE_MENTION):]189    if not _UUID_RE.fullmatch(text):190        raise NodeError(191            f"{label} is not a Drive file id (expected a UUID or mention://drive_file/<uuid>)",192            "invalid_input",193        )194    return text195196197def _language(value: Any, label: str, default: str, *, allow_auto: bool = False) -> str:198    """A supported language code (or "auto" where allowed); the default when not given."""199    if value is None or (isinstance(value, str) and not value.strip()):200        return default201    key = value.strip().lower() if isinstance(value, str) else ""202    if allow_auto and key in (AUTO, "auto-detect", "detect"):203        return AUTO204    match = _LANGUAGE_RE.fullmatch(key)205    code = _LANGUAGE_BY_NAME.get(key) or (match.group(1) if match else None)206    if code not in LANGUAGES:207        raise NodeError(208            f"{label} must be one of {', '.join(LANGUAGES)}", "invalid_input"209        )210    return code211212213def _pages(value: Any) -> str | None:214    """The page selection without whitespace ("1-5,8"); None for every page."""215    if value is None:216        return None217    if isinstance(value, int) and not isinstance(value, bool):218        value = str(value)219    if not isinstance(value, str):220        raise NodeError(_PAGES_FORMAT, "invalid_input")221    spec = re.sub(r"\s+", "", value)222    if not spec:223        return None224    if not _PAGES_RE.fullmatch(spec):225        raise NodeError(_PAGES_FORMAT, "invalid_input")226    for part in spec.split(","):227        first, _, last = part.partition("-")228        start = int(first)229        end = int(last) if last else start230        if start < 1 or end < start:231            raise NodeError(_PAGES_FORMAT, "invalid_input")232    return spec233234235def _flag(value: Any, *, default: bool = False) -> bool:236    """A boolean input; missing or empty (an edge with no value) is ``default``."""237    if value is None or (isinstance(value, str) and not value.strip()):238        return default239    if isinstance(value, str):240        return value.strip().lower() in ("true", "1", "yes", "on")241    if isinstance(value, (bool, int)):242        return bool(value)243    return False244245246def _style(value: Any) -> str | None:247    """The free-text style, or None when empty; the adapter caps its length."""248    if value is None:249        return None250    if not isinstance(value, str):251        raise NodeError("Style must be text", "invalid_input")252    return value.strip() or None253254255def _model_id(value: Any) -> str | None:256    """The llm-picker selection; the planner fills an empty one with the default."""257    return value.strip() if isinstance(value, str) and value.strip() else None258259260# ── Drive ───────────────────────────────────────────────────────────────────261262263def _clean(text: Any) -> str:264    """One line, clipped; empty when it carries a URL (a presigned one is a credential)."""265    if not isinstance(text, str):266        return ""267    line = " ".join(text.split())268    return "" if "://" in line else line[:_MAX_MESSAGE_CHARS]269270271def _route_failure(exc: RemoteError) -> tuple[str, str | None]:272    """The www route's own ``{message, errorCode}`` when the body is that JSON.273274    Route messages are written by www-emblema and hold no document text. Any275    other body (a storage error page) is not shown.276    """277    message, code = "", None278    try:279        body = json.loads(exc.stderr) if isinstance(exc.stderr, str) else None280    except ValueError:281        body = None282    if isinstance(body, dict):283        message = _clean(body.get("message"))284        raw_code = body.get("errorCode")285        if isinstance(raw_code, str) and _ROUTE_CODE_RE.fullmatch(raw_code):286            code = raw_code287    if not message and exc.args:288        # The SDK may carry the route's message as the exception's own; its289        # generic "GET <url> failed" is dropped by _clean (it holds a URL).290        message = _clean(exc.args[0])291    return message, code292293294def _drive_failure(exc: GaisError, label: str) -> NodeError:295    prefix = f"{label}: " if label else ""296    if isinstance(exc, RemoteError):297        status = exc.http_status298        message, code = _route_failure(exc)299        if status == 422:300            return NodeError(prefix + (message or _NO_PDF), code or "NO_PDF")301        if status in (401, 403):302            return NodeError(303                prefix + (message or "You are not allowed to download this Drive file"),304                code or "DOWNLOAD_FORBIDDEN",305            )306        if status == 404:307            return NodeError(prefix + (message or "Drive file not found"), code or "ITEM_NOT_FOUND")308        detail = f": {message}" if message else ""309        return NodeError(310            f"{prefix}Drive could not provide the file (HTTP {status}){detail}",311            code or "drive_unavailable",312        )313    if isinstance(exc, BackendTimeout):314        return NodeError(315            f"{prefix}Downloading the file from Drive timed out; try again", "drive_timeout"316        )317    if isinstance(exc, InvalidParams):318        return NodeError(f"{prefix}Not a Drive file id", "invalid_input")319    detail = _clean(str(exc)) or type(exc).__name__320    return NodeError(321        f"{prefix}Could not download the file from Drive ({detail})", "drive_unavailable"322    )323324325def _staged_file(result: Any, label: str) -> str:326    path = (getattr(result, "files", None) or {}).get("file")327    if not path:328        prefix = f"{label}: " if label else ""329        raise NodeError(f"{prefix}Drive returned no file", "drive_unavailable")330    return str(path)331332333def _download_pdf(item_id: str) -> tuple[str, dict[str, Any]]:334    """The item's PDF, staged by gais, and the download metadata."""335    try:336        result = Gais.drive.download(item_id, as_pdf=True)337    except TypeError as exc:338        # An executor image older than the PDF Translator: no as_pdf in its SDK.339        if "as_pdf" in str(exc):340            raise NodeError(_REBUILD_EXECUTOR, "babeldoc_missing") from None341        raise342    except GaisError as exc:343        raise _drive_failure(exc, "") from None344    return _staged_file(result, ""), dict(getattr(result, "metadata", None) or {})345346347def _require_pdf(path: str) -> None:348    with open(path, "rb") as fh:349        head = fh.read(_PDF_HEADER_WINDOW)350    if _PDF_MAGIC not in head:351        raise NodeError(_NOT_A_PDF, "not_a_pdf")352353354def _display_name(value: Any) -> str:355    """A Drive file name for the report: one line, clipped."""356    text = _CONTROL_CHARS.sub(" ", value) if isinstance(value, str) else ""357    return " ".join(text.split())[:_MAX_NAME_CHARS]358359360def _glossary_kind(name: str, mime_type: str) -> str:361    extension = name.rsplit(".", 1)[1].lower() if "." in name else ""362    if extension in _GLOSSARY_KINDS:363        return extension364    kind = _GLOSSARY_MIME_KINDS.get(mime_type.split(";", 1)[0].strip().lower())365    if kind:366        return kind367    shown = f" (this one is .{extension})" if re.fullmatch(r"[a-z0-9]{1,10}", extension) else ""368    raise NodeError(f"Glossary file must be a CSV or XLSX file{shown}", "glossary_invalid")369370371def _short_hash(value: str) -> str:372    """The first 12 characters of a content hash, without an ``algo:`` prefix."""373    return value.rsplit(":", 1)[-1].strip()[:12]374375376def _glossary(item_id: str, scratch_dir: str) -> tuple[str, str]:377    """Fetch the glossary; return its path for the adapter and the report line."""378    try:379        result = Gais.drive.download(item_id)380    except GaisError as exc:381        raise _drive_failure(exc, "Glossary file") from None382    source = _staged_file(result, "Glossary file")383    metadata = dict(getattr(result, "metadata", None) or {})384    name = _display_name(metadata.get("name"))385    kind = _glossary_kind(name, str(metadata.get("mimeType") or ""))386    path = f"{scratch_dir}/glossary.{kind}"387    digest = _copy_file(source, path)388    drive_hash = str(metadata.get("hash") or "").strip()389    fingerprint = f"hash {_short_hash(drive_hash)}" if drive_hash else f"sha256 {digest[:12]}"390    return path, f"Glossary: {name or 'unnamed file'} ({fingerprint})"391392393# ── translation ─────────────────────────────────────────────────────────────394395396def _translate(pdf_path: str, **kwargs: Any) -> Any:397    """``Gais.pdf_translate.translate``, with its failures made node errors."""398    try:399        adapter = Gais.pdf_translate400    except (AttributeError, ImportError):401        # The executor image predates the adapter (or lost one of its modules).402        raise NodeError(_REBUILD_EXECUTOR, "babeldoc_missing") from None403    try:404        return adapter.translate(pdf_path, **kwargs)405    except adapter.PdfTranslateError as exc:406        # Content-free by the adapter's contract. One line: the worker shows407        # only the last line of a multi-line error.408        message = "; ".join(line.strip() for line in str(exc).splitlines() if line.strip())409        failure_class = getattr(exc, "failure_class", None) or "pdf_translate_failed"410        raise NodeError(message or "The translation failed", str(failure_class)) from None411    except InvalidParams as exc:412        # An argument the PDF itself refutes ("pages selects no page of this413        # 3-page PDF"). The adapter's argument messages name no document text.414        raise NodeError(_clean(str(exc)) or "The translation's inputs were refused", "invalid_input") from None415416417def _result_file(result: Any, key: str, label: str) -> str:418    path = (getattr(result, "files", None) or {}).get(key)419    if not path:420        raise NodeError(f"BabelDOC finished without the {label}", "babeldoc_crash")421    return str(path)422423424# ── output ──────────────────────────────────────────────────────────────────425426427def _output_stem(name: Any) -> str:428    """The source file's name, made a safe file-name stem."""429    base = str(name or "").replace("\\", "/").rsplit("/", 1)[-1].strip()430    stem = _UNSAFE_NAME_CHARS.sub(" ", _SOURCE_EXTENSION.sub("", base))431    stem = re.sub(r"\s+", " ", stem).lstrip(". ").rstrip()[:_MAX_NAME_CHARS]432    while len(stem.encode("utf-8")) > _MAX_NAME_BYTES:433        stem = stem[:-1]434    return stem.rstrip(" .") or DEFAULT_NAME435436437def _copy_file(src: str, dst: str) -> str:438    """Stream-copy ``src`` to ``dst``; return the content's sha256."""439    digest = hashlib.sha256()440    with open(src, "rb") as fin, open(dst, "wb") as fout:441        while chunk := fin.read(_COPY_CHUNK):442            digest.update(chunk)443            fout.write(chunk)444    return digest.hexdigest()445446447# ── run ─────────────────────────────────────────────────────────────────────448449450def process(inputs: dict[str, Any]) -> dict[str, Any]:451    item_id = _drive_item_id(inputs.get("driveItemId"), "Document")452    if item_id is None:453        raise NodeError(454            "Required input 'driveItemId' not provided: pick the Drive file to translate",455            "invalid_input",456        )457    target = _language(inputs.get("targetLanguage"), "Target Language", DEFAULT_TARGET)458    source = _language(inputs.get("sourceLanguage"), "Source Language", DEFAULT_SOURCE, allow_auto=True)459    if source == target:460        raise NodeError(461            f"Source and target language are both {LANGUAGES[target]}; "462            "pick a different target language",463            "invalid_input",464        )465    pages = _pages(inputs.get("pages"))466    # On unless switched off: the original page beside its translation lets a467    # reviewer check every line, and keeps stamps and signatures as they are.468    bilingual = _flag(inputs.get("bilingual"), default=True)469    style = _style(inputs.get("style"))470    model_id = _model_id(inputs.get("llmModelId"))471    glossary_id = _drive_item_id(inputs.get("glossary"), "Glossary")472473    pdf_path, pdf_metadata = _download_pdf(item_id)474    _require_pdf(pdf_path)475476    glossary_path, glossary_line = None, ""477    if glossary_id:478        glossary_dir = tempfile.mkdtemp(prefix="pdf-translator-glossary-", dir=SCRATCH_DIR)479        glossary_path, glossary_line = _glossary(glossary_id, glossary_dir)480481    result = _translate(482        pdf_path,483        target_lang=target,484        source_lang=source,485        model_id=model_id,486        pages=pages,487        bilingual=bilingual,488        glossary_path=glossary_path,489        out_dir=tempfile.mkdtemp(prefix="pdf-translator-out-", dir=SCRATCH_DIR),490        style=style,491    )492    translated_src = _result_file(result, "translated", "translated PDF")493    bilingual_src = _result_file(result, "bilingual", "bilingual PDF") if bilingual else None494495    stem = _output_stem(pdf_metadata.get("name"))496    translated_name = f"{stem}.{target}.pdf"497    _copy_file(translated_src, f"{OUTPUT_DIR}/{translated_name}")498    bilingual_name = None499    if bilingual_src:500        bilingual_name = f"{stem}.{target}.bilingual.pdf"501        _copy_file(bilingual_src, f"{OUTPUT_DIR}/{bilingual_name}")502503    report = (getattr(result, "text", None) or "").strip() or (504        f"Translated from {LANGUAGES.get(source, 'the detected language')} to {LANGUAGES[target]}."505    )506    if glossary_line:507        report = f"{report}\n{glossary_line}"508    return {509        "translatedPdf": translated_name,510        "bilingualPdf": bilingual_name,511        "report": report,512    }513514515def _fail(payload: dict[str, Any]) -> None:516    print(json.dumps(payload), file=sys.stderr)517    sys.exit(1)518519520def main() -> None:521    try:522        envelope = json.loads(sys.stdin.read() or "{}")523        if not isinstance(envelope, dict):524            envelope = {}525        inputs = envelope.get("inputs") or {}526        output = process(inputs if isinstance(inputs, dict) else {})527    except NodeError as exc:528        _fail({"error": str(exc), "errorType": exc.error_type})529    except Exception as exc:  # noqa: BLE001 — every other failure, message withheld (C5)530        _fail({531            "error": (532                f"Unexpected failure in the PDF Translator ({type(exc).__name__}). "533                "Its details are withheld so no document text reaches the logs"534            ),535            "errorType": type(exc).__name__,536            "traceback": "".join(traceback.format_tb(exc.__traceback__)),537        })538    else:539        json.dump(output, sys.stdout)540541542if __name__ == "__main__":543    main()544