Codebase map

1"""Codebase map — what already exists, read from the repository itself.23gstack's reviews run INSIDE the repository. Their first question is the eng4review's scope challenge — "what existing code already solves each5sub-problem" — and office-hours asks the same of every premise. A Plan review6node sees only the design doc's text, so without this node its "already7exists" and every file path it names are guesses. This node is where they8stop being guesses.910It reads the repository ONCE, at the plan's working branch, and every review11reads its map. One grounded retrieval instead of five that would disagree, and12a readable artefact on the canvas that shows exactly what the reviews knew.1314## How it chooses what to read1516It cannot read a monorepo, and it does not try:1718  1. the tree at the branch (one API call — paths only);19  2. a model turns the plan into search terms;20  3. top-level structure + the paths matching those terms become a shortlist;21  4. a model picks up to `maxFiles` from the shortlist — picks outside the22     tree are dropped, so it cannot "read" a file that does not exist;23  5. those files and the convention files (CLAUDE.md, AGENTS.md, README...)24     are read, each and in total size-capped;25  6. a model writes the map, citing only paths it read or saw in the tree.2627GitHub is reached only through `gais.connectors.github` — the sandbox blocks28network imports, and the connector keeps the token out of every error.2930## It runs twice, and it states facts (2026-09-29)3132gstack's /office-hours opens with "Context Gathering" — CLAUDE.md, `git log`,33`git diff --stat`, a grep of the request's areas — BEFORE its first question.34So the map now runs first at step 4, right after the Working branch, against35the user's REQUEST (the design doc holds just the request until it is36written), and again in Phase 4 against the written doc: the doc's edge makes37it out of date. A short-lived "Repo probe" node did the first read; it was38folded back in here, because the map was the better read.3940Beside the model's map it states FACTS, written by code, never paraphrased by41a model — a check command or a PR number is either there or it is not:42  - the repository: size, languages, top level, convention files, empty or not;43  - the branch: commits beyond its base, pull requests open into and out of it;44  - the checks: package.json scripts (with the lockfile's package manager),45    Makefile targets, check.sh, CI workflows.46The agent reads `facts` for the branch-state question and the approval gate's47checks. A repository with a dozen files or fewer is read whole, with one call.48"""4950import asyncio51import json52import posixpath53import re54import sys55import traceback56from typing import Any5758from gais import Gais, connectors59from gais import github_repo as gh60from gais._errors import ConfigError, RemoteError6162#: Files that state a repository's rules. Read first and always when present:63#: a plan that breaks a convention written in CLAUDE.md is a finding every64#: review should be able to make.65CONVENTION_NAMES = ("CLAUDE.md", "AGENTS.md", "CONTRIBUTING.md", "README.md", "DESIGN.md")6667#: Never worth a slot on the shortlist.68NOISE = re.compile(69    r"(^|/)(node_modules|dist|build|\.next|coverage|vendor|__pycache__|\.git|\.venv)(/|$)"70    r"|\.(png|jpe?g|gif|svg|ico|webp|pdf|zip|gz|woff2?|ttf|mp[34]|lock|min\.js|map)$"71    r"|(^|/)(pnpm-lock\.yaml|package-lock\.json|yarn\.lock|uv\.lock|poetry\.lock)$",72    re.IGNORECASE,73)7475SHORTLIST_LIMIT = 40076STRUCTURE_DEPTH = 277FILE_CHARS = 8_00078TOTAL_CHARS = 120_00079NESTED_CONVENTIONS = 480SMALL_REPO = 12          # content files: read them all, no search step81TOP_LEVEL_LIMIT = 4082COMMITS_SHOWN = 108384#: Root files that say what a repository can build, test and run.85MANIFEST_NAMES = ("package.json", "Makefile", "pyproject.toml", "requirements.txt", "go.mod",86                  "Cargo.toml", "Gemfile", "composer.json", "Dockerfile", "docker-compose.yml",87                  "docker-compose.yaml", "check.sh")8889#: Files that do not make a repository "have code": one holding only these is90#: empty for planning purposes.91BOILERPLATE = re.compile(92    r"^(README|LICENSE|LICENCE|CHANGELOG|CODEOWNERS|CONTRIBUTING|SECURITY)(\.[\w]+)?$"93    r"|^\.(gitignore|gitattributes|editorconfig|nvmrc|node-version)$|^\.github/",94    re.IGNORECASE,95)9697LANGUAGES = {98    ".py": "Python", ".ts": "TypeScript", ".tsx": "TypeScript", ".js": "JavaScript",99    ".jsx": "JavaScript", ".mjs": "JavaScript", ".cjs": "JavaScript", ".html": "HTML",100    ".css": "CSS", ".scss": "CSS", ".go": "Go", ".rs": "Rust", ".rb": "Ruby", ".java": "Java",101    ".kt": "Kotlin", ".php": "PHP", ".cs": "C#", ".swift": "Swift", ".vue": "Vue",102    ".svelte": "Svelte", ".sql": "SQL", ".sh": "Shell",103}104105106def is_noise(path: str) -> bool:107    return bool(NOISE.search(path))108109110#: A branch name as one path segment — `vibe/x` must not become two. Shared111#: with the rest of the repository node set (`gais.github_repo`).112enc = gh.enc113114115# ── Facts: read, never inferred ─────────────────────────────────────────────116117def is_boilerplate(path: str) -> bool:118    return bool(BOILERPLATE.search(path))119120121def languages(blobs: list[str]) -> dict[str, int]:122    counts: dict[str, int] = {}123    for p in blobs:124        name = None if is_noise(p) else LANGUAGES.get(posixpath.splitext(p)[1].lower())125        if name:126            counts[name] = counts.get(name, 0) + 1127    return dict(sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])))128129130def top_level(blobs: list[str]) -> list[str]:131    seen: list[str] = []132    for p in sorted(blobs):133        entry = p.split("/", 1)[0] + ("/" if "/" in p else "")134        if entry not in seen and not is_noise(entry):135            seen.append(entry)136    return seen[:TOP_LEVEL_LIMIT]137138139def package_manager(blobs: list[str]) -> str:140    names = set(blobs)141    if "pnpm-lock.yaml" in names:142        return "pnpm"143    if "yarn.lock" in names:144        return "yarn"145    if "bun.lockb" in names or "bun.lock" in names:146        return "bun"147    return "npm"148149150def package_scripts(text: str, manager: str) -> list[dict]:151    try:152        scripts = (json.loads(text) or {}).get("scripts") or {}153    except (json.JSONDecodeError, ValueError, AttributeError):154        return []155    run = "yarn" if manager == "yarn" else f"{manager} run"156    return [{"source": "package.json", "name": name, "command": f"{run} {name}"}157            for name in scripts if isinstance(name, str)]158159160MAKE_TARGET = re.compile(r"^([A-Za-z0-9][\w.-]*)\s*:(?!=)", re.MULTILINE)161162163def make_targets(text: str) -> list[dict]:164    names: list[str] = []165    for name in MAKE_TARGET.findall(text or ""):166        if name not in names:167            names.append(name)168    return [{"source": "Makefile", "name": n, "command": f"make {n}"} for n in names]169170171def headline(commit: Any) -> str:172    return (((commit or {}).get("commit") or {}).get("message") or "").split("\n", 1)[0] \173        if isinstance(commit, dict) else ""174175176def branch_state(owner: str, repo: str, base: str, branch: str, get) -> dict:177    """What the branch holds beyond its base, and what is open against it.178179    Never fatal: a map without branch facts is still a map.180    """181    path = f"/repos/{owner}/{repo}"182    state: dict[str, Any] = {"branch": branch, "base": base}183    if base and base != branch:184        try:185            cmp = get(f"{path}/compare/{enc(base)}...{enc(branch)}")186        except RemoteError as exc:187            cmp = None188            state["compare_error"] = (f"Could not compare {branch} with {base} "189                                      f"(HTTP {getattr(exc, 'http_status', '?')}).")190        if isinstance(cmp, dict):191            commits = cmp.get("commits") or []192            state.update({193                "ahead_by": cmp.get("ahead_by", len(commits)),194                "behind_by": cmp.get("behind_by", 0),195                "commits": [headline(c) for c in commits][-COMMITS_SHOWN:],196                "changed_files": [f.get("filename") for f in (cmp.get("files") or [])197                                  if isinstance(f, dict)][:50],198            })199200    def prs(query: str) -> list[dict]:201        try:202            rows = get(f"{path}/pulls?state=open&per_page=20&{query}")203        except RemoteError:204            return []205        return [{"number": r.get("number"), "title": r.get("title") or "",206                 "head": (r.get("head") or {}).get("ref"), "base": (r.get("base") or {}).get("ref"),207                 "draft": bool(r.get("draft")), "url": r.get("html_url") or ""}208                for r in rows if isinstance(r, dict)] if isinstance(rows, list) else []209210    state["open_prs_into"] = prs(f"base={enc(branch)}")211    state["open_prs_from"] = prs(f"head={owner}:{enc(branch)}")212    try:213        recent = get(f"{path}/commits?sha={enc(branch)}&per_page=5")214    except RemoteError:215        recent = []216    state["recent_commits"] = [headline(c) for c in recent] if isinstance(recent, list) else []217    return state218219220def repo_facts(owner: str, repo: str, ref: str, base: str, blobs: list[str], truncated: bool,221               get, read) -> dict:222    """Everything the map states without a model."""223    if not base:224        try:225            info = get(f"/repos/{owner}/{repo}")226            base = info.get("default_branch") or "" if isinstance(info, dict) else ""227        except RemoteError:228            base = ""229    manifests = [p for p in blobs if p in MANIFEST_NAMES]230    scripts: list[dict] = []231    for name, parse in (("package.json", lambda t: package_scripts(t, package_manager(blobs))),232                        ("Makefile", make_targets)):233        if name in manifests:234            try:235                text = read(f"/repos/{owner}/{repo}/contents/{name}?ref={enc(ref)}")236            except RemoteError:237                continue238            scripts += parse(text if isinstance(text, str) else "")239    if "check.sh" in manifests:240        scripts.append({"source": "check.sh", "name": "check.sh", "command": "bash check.sh"})241    return {242        "ref": ref,243        "base": base,244        "empty": not any(not is_boilerplate(p) and not is_noise(p) for p in blobs),245        "file_count": len(blobs),246        "truncated": truncated,247        "languages": languages(blobs),248        "top_level": top_level(blobs),249        "conventions": [p for p in blobs if posixpath.basename(p) in CONVENTION_NAMES250                        and p.count("/") <= 2][:10],251        "manifests": manifests,252        "scripts": scripts,253        "workflows": sorted(p for p in blobs if p.startswith(".github/workflows/")),254        "branch": branch_state(owner, repo, base, ref, get),255    }256257258def render_facts(facts: dict) -> str:259    """The facts as markdown — code, not a model, writes these three sections."""260    langs = ", ".join(f"{k} {v}" for k, v in facts["languages"].items()) or "none recognised"261    lines = ["## Repository", f"- {facts['file_count']} files at `{facts['ref']}`; languages: {langs}."]262    if facts["empty"]:263        lines.append("- It holds no code yet — only "264                     + (", ".join(f"`{p}`" for p in facts["top_level"]) or "nothing") + ".")265    else:266        lines.append("- Top level: " + ", ".join(f"`{p}`" for p in facts["top_level"]) + ".")267    if facts["conventions"]:268        lines.append("- Convention files: " + ", ".join(f"`{p}`" for p in facts["conventions"]) + ".")269270    b = facts["branch"]271    lines += ["", "## Branch"]272    if b.get("compare_error"):273        lines.append(f"- {b['compare_error']}")274    elif b.get("base") and b.get("base") != b.get("branch"):275        ahead = b.get("ahead_by") or 0276        if ahead:277            lines.append(f"- `{b['branch']}` is {ahead} commit{'s' if ahead != 1 else ''} ahead of "278                         f"`{b['base']}` ({b.get('behind_by', 0)} behind): "279                         + "; ".join(b.get("commits") or []) + ".")280            if b.get("changed_files"):281                lines.append("- Changed beyond the base: "282                             + ", ".join(f"`{p}`" for p in b["changed_files"]) + ".")283        else:284            lines.append(f"- `{b['branch']}` holds nothing beyond `{b['base']}`.")285    for label, key in (("Open pull requests INTO it", "open_prs_into"),286                       ("Open pull requests FROM it", "open_prs_from")):287        rows = b.get(key) or []288        if rows:289            lines.append(f"- {label}: " + "; ".join(290                f"#{r['number']} {r['title']} (`{r['head']}` → `{r['base']}`{', draft' if r['draft'] else ''})"291                for r in rows) + ".")292    if b.get("recent_commits"):293        lines.append("- Latest commits: " + "; ".join(b["recent_commits"]) + ".")294295    lines += ["", "## Checks the repository has"]296    if facts["scripts"] or facts["workflows"]:297        lines += [f"- `{x['command']}` ({x['source']})" for x in facts["scripts"]]298        lines += [f"- CI: `{p}`" for p in facts["workflows"]]299    else:300        lines.append("- None: no package.json scripts, no Makefile, no check script, no CI workflow.")301    if facts["truncated"]:302        lines += ["", "NOTE: the repository is larger than GitHub lists in one call; "303                      "paths outside the listing were not considered."]304    return "\n".join(lines)305306307def conventions_in(blobs: list[str], picked_dirs: set[str]) -> list[str]:308    """Root convention files, plus CLAUDE.md / AGENTS.md in the directories309    the picked files live in — this repo keeps per-app rules next to the app."""310    root = [p for p in blobs if p in CONVENTION_NAMES]311    nested = sorted(312        p for p in blobs313        if "/" in p and p.rsplit("/", 1)[1] in ("CLAUDE.md", "AGENTS.md")314        and any(d == p.rsplit("/", 1)[0] or d.startswith(p.rsplit("/", 1)[0] + "/") for d in picked_dirs)315    )316    return sorted(root, key=CONVENTION_NAMES.index) + nested[:NESTED_CONVENTIONS]317318319def structure(blobs: list[str], depth: int = STRUCTURE_DEPTH) -> list[str]:320    """Directories down to `depth` — the shape of the repository, for the picker."""321    dirs = set()322    for p in blobs:323        parts = p.split("/")[:-1]324        for i in range(1, min(len(parts), depth) + 1):325            dirs.add("/".join(parts[:i]) + "/")326    return sorted(dirs)327328329def shortlist(blobs: list[str], terms: list[str], limit: int = SHORTLIST_LIMIT) -> list[str]:330    """Paths matching any search term, most matches first, then shallowest.331332    Matching is on the path, lower-cased: a term like `share` finds333    `share-link-dialog.tsx` and `lib/share-links/`. Crude, and deliberately so334    — the model picks from this list, so it only has to contain the right335    files, not rank them perfectly.336    """337    terms = [t.lower() for t in terms if isinstance(t, str) and len(t.strip()) >= 3]338    scored = []339    for p in blobs:340        if is_noise(p):341            continue342        low = p.lower()343        hits = sum(1 for t in terms if t in low)344        if hits:345            scored.append((-hits, p.count("/"), p))346    return [p for _, _, p in sorted(scored)[:limit]]347348349def keep_real(picks: Any, blobs: set[str], limit: int) -> list[str]:350    """The model's picks that are real files, in order, de-duplicated."""351    out = []352    for p in picks if isinstance(picks, list) else []:353        if isinstance(p, str) and p in blobs and p not in out and not is_noise(p):354            out.append(p)355        if len(out) >= limit:356            break357    return out358359360def parse_json(raw: str) -> dict:361    text = raw.strip()362    if text.startswith("```"):363        text = text.split("\n", 1)[-1].rstrip().removesuffix("```")364    try:365        data = json.loads(text)366        return data if isinstance(data, dict) else {}367    except (json.JSONDecodeError, ValueError):368        match = re.search(r"\{.*\}", text, re.DOTALL)369        if match:370            try:371                data = json.loads(match.group(0))372                return data if isinstance(data, dict) else {}373            except (json.JSONDecodeError, ValueError):374                pass375    return {}376377378def read_files(owner: str, repo: str, ref: str, paths: list[str], read) -> list[tuple[str, str]]:379    """(path, content) for each path, each and in total size-capped."""380    out, total = [], 0381    for path in paths:382        if total >= TOTAL_CHARS:383            break384        try:385            text = read(f"/repos/{owner}/{repo}/contents/{gh.enc_path(path)}?ref={enc(ref)}")386        except RemoteError:387            continue  # a file that vanished or is too large for the API: skip it388        if not isinstance(text, str):389            continue390        if len(text) > FILE_CHARS:391            text = text[:FILE_CHARS] + f"\n… [truncated at {FILE_CHARS} characters]"392        text = text[: TOTAL_CHARS - total]393        total += len(text)394        out.append((path, text))395    return out396397398async def ask(model_id: str, system: str, user: str) -> str:399    response = await Gais.llm.chat_async(400        [{"role": "system", "content": system}, {"role": "user", "content": user}],401        model_id=model_id, temperature=0, thinking=False)402    return response.text403404405TERMS_PROMPT = """You turn a software plan into repository search terms.406Return ONLY JSON: {"terms": ["...", "..."]} — 6 to 15 short lowercase terms407that would appear in the FILE PATHS of code this plan touches or should reuse:408domain nouns, feature names, likely module or directory names. No sentences."""409410PICK_PROMPT = """You choose which files of a repository to read for a plan.411You see the repository's structure and a shortlist of paths. Return ONLY JSON:412{"files": ["path", ...]} with at most {limit} paths, copied EXACTLY from the413shortlist, most important first. Prefer: code the plan would reuse or change,414the entry points of the areas it touches, existing tests for those areas.415Never invent a path."""416417MAP_PROMPT = """You write a CODEBASE MAP: what a team must know about an418existing repository before building a plan in it. Before the plan is written,419the <plan> is just the user's request: map against that. Use ONLY the files provided420and the paths listed; never invent a path, a function or a convention.421422Write markdown with exactly these sections:423424## Conventions425Rules from the repository's own convention files that this plan must follow,426each with the file it comes from. Skip the section's items if none apply.427428## What already exists429For each part of the plan, the existing code that already does some or all of430it: `path` — what it does — how the plan should reuse it. Say plainly when a431part of the plan has nothing.432433## What does not exist yet434The parts of the plan that are genuinely new.435436## Where it touches shared code437Existing modules the plan would change that other features also depend on.438439Be specific and short. A reviewer will use this to judge the plan, so a wrong440path is worse than a missing one."""441442443async def build_map(owner, repo, ref, focus, max_files, model_id, call=None, base=""):444    read = call or (lambda path: connectors.github.request("GET", path, raw=True))445    get = call or (lambda path: connectors.github.request("GET", path))446447    tree = get(f"/repos/{owner}/{repo}/git/trees/{enc(ref)}?recursive=1")448    entries = tree.get("tree") if isinstance(tree, dict) else None449    if not isinstance(entries, list):450        raise ValueError(f"Could not list {owner}/{repo} at '{ref}'.")451    blobs = [e["path"] for e in entries if isinstance(e, dict) and e.get("type") == "blob" and e.get("path")]452    facts = repo_facts(owner, repo, ref, base, blobs, bool(tree.get("truncated")), get, read)453454    content = [p for p in blobs if not is_noise(p)]455    if len(content) <= SMALL_REPO:456        # Nothing to search: read it all, with no terms or pick call.457        chosen, candidates = [p for p in content if p not in CONVENTION_NAMES], content458    else:459        terms = parse_json(await ask(model_id, TERMS_PROMPT, focus)).get("terms") or []460        candidates = shortlist(blobs, terms)461        picks = parse_json(await ask(462            model_id, PICK_PROMPT.replace("{limit}", str(max_files)),463            f"<plan>\n{focus}\n</plan>\n\n<structure>\n" + "\n".join(structure(blobs))464            + "\n</structure>\n\n<shortlist>\n" + "\n".join(candidates) + "\n</shortlist>",465        )).get("files")466        chosen = keep_real(picks, set(blobs), max_files)467    picked_dirs = {p.rsplit("/", 1)[0] for p in chosen if "/" in p}468    conv = conventions_in(blobs, picked_dirs)469    files = read_files(owner, repo, ref, conv + [p for p in chosen if p not in conv], read)470471    body = "\n\n".join(f"<file path=\"{p}\">\n{t}\n</file>" for p, t in files)472    written = (await ask(473        model_id, MAP_PROMPT,474        f"<plan>\n{focus}\n</plan>\n\n<paths-seen>\n" + "\n".join(candidates[:200])475        + f"\n</paths-seen>\n\n{body}",476    )).strip()477    return {"map": f"{written}\n\n{render_facts(facts)}", "files": [p for p, _ in files],478            "ref": ref, "facts": facts}479480481def main():482    try:483        envelope = json.loads(sys.stdin.read() or "{}")484        inputs = envelope.get("inputs", {}) if isinstance(envelope, dict) else {}485        repo_in = inputs.get("repo") if isinstance(inputs.get("repo"), dict) else {}486        owner, repo = (repo_in.get("owner") or "").strip(), (repo_in.get("repo") or "").strip()487        ref = (repo_in.get("base_branch") or "").strip()488        base = str(inputs.get("base") or "").strip()489        focus = (inputs.get("focus") or "").strip()490        model_id = inputs.get("llmModelId")491        try:492            max_files = max(5, min(40, int(inputs.get("maxFiles") or 20)))493        except (TypeError, ValueError):494            max_files = 20495496        if not owner or not repo or not ref:497            raise ValueError("Wire the Working branch's `repo` output into Repository — owner, repo and branch are needed.")498        if not focus:499            raise ValueError("Nothing to map against: write the user's request (or the design doc) into the Design doc wired to Plan.")500        if not model_id:501            raise ValueError("Required input 'llmModelId' not provided")502        connectors.github.require()503504        print(f"[codebase-map] {owner}/{repo}@{ref} base={base or '(default)'} max_files={max_files}", file=sys.stderr)505        out = asyncio.run(build_map(owner, repo, ref, focus, max_files, model_id, base=base))506        b = out["facts"]["branch"]507        print(f"[codebase-map] read {len(out['files'])} files empty={out['facts']['empty']} "508              f"ahead={b.get('ahead_by')} prs_into={len(b.get('open_prs_into') or [])}", file=sys.stderr)509        json.dump(out, sys.stdout)510    except (ConfigError, RemoteError, ValueError) as e:511        print(json.dumps({"error": str(e), "errorType": type(e).__name__}), file=sys.stderr)512        sys.exit(1)513    except Exception as e:514        print(json.dumps({"error": str(e), "errorType": type(e).__name__,515                          "traceback": traceback.format_exc()}), file=sys.stderr)516        sys.exit(1)517518519if __name__ == "__main__":520    main()521