The Grounding Pass

#!/usr/bin/env python3
"""grounding_pass.py: find the notes your AI wrote that reach no source.

An orphan checker asks "does anything link TO this note." That is link presence, and it is a
1-hop question any query can answer. This asks a different, multi-hop question: "following this
note's outbound links, do you ever arrive at a SOURCE, a note with real provenance?" A note can
link out to five other notes and still reach no source, because every one of them is itself
derived. That note is a source-orphan: well-connected, trusted, resting on nothing. It is the
exact note an LLM maintaining your vault produces, and no orphan tool can see it.

The check needs no model, makes no network call, and never writes to your notes. It reads them,
walks the link graph, and prints the source-orphans. It appends one line, a date and the counts,
to a local log so you can watch the number fall week on week.

    python3 grounding_pass.py /path/to/your/vault

Classification (the one rule everything rests on, tune it to your vault in the flags):
  A note is a SOURCE if either
    - its frontmatter carries provenance: a non-empty `url`, `source`, or `source_url`; or
    - it lives in a folder you name as source material (default: Sources, Clippings).
  A bare http link in the body does NOT make a note a source. An agent-written summary cites URLs
  all the time; blessing it as a source would let the very note this tool hunts launder itself.
  Everything else is DERIVED. Notes that are legitimately sourceless navigation (MOCs, hubs, daily
  notes, templates) are excluded from the report by folder, and counted separately.

Grounding is a distance. A note is grounded if a source sits within N outbound hops of it
(default 2, which allows the legitimate note -> literature-note -> source chain). --strict sets
N=1 and demands a direct link to a source; raise --max-hops to be more forgiving.

Two blind spots it will tell you about, and cannot fix:
  1. It checks that a source is REACHED, not that the source SUPPORTS the claim. That is a
     semantic question and needs a model.
  2. A larger N launders grounding through chains of derived notes. N=2 is the honest default;
     --strict (N=1) is the aggressive read that trusts nothing but a direct citation.

Zero dependencies. Python 3.8+.
"""
from __future__ import annotations

import argparse
import datetime as _dt
import json
import os
import re
import sys
WIKILINK = re.compile(r"\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]")
# a markdown link to a .md note, allowing an optional #anchor or ?query after .md
MDLINK = re.compile(r"(?<!!)\[[^\]]*\]\(([^)#?]+\.md)(?:[#?][^)]*)?\)")
# a provenance field and its value ON THE SAME LINE. [^\S\n] is horizontal whitespace
# only, so an empty `url:` followed by another key does NOT capture the next line's
# value (which would misclassify an ungrounded note as a source).
FM_KEY = re.compile(r"^(url|source|source_url)[^\S\n]*:[^\S\n]*(\S.*?)?[^\S\n]*$", re.I | re.M)


def split_frontmatter(text: str) -> tuple[str, str]:
    """Return (frontmatter, body). Frontmatter is the leading --- ... --- block, else ''."""
    if text.startswith("---"):
        end = text.find("\n---", 3)
        if end != -1:
            nl = text.find("\n", end + 1)
            return text[3:end], text[nl + 1:] if nl != -1 else ""
    return "", text


def has_provenance(fm: str) -> bool:
    for m in FM_KEY.finditer(fm):
        val = (m.group(2) or "").strip().strip("'\"")
        if val and val.lower() not in ("''", '""', "none", "null", "~", "[]"):
            return True
    return False


def in_any(path_parts: set[str], folders: list[str]) -> bool:
    return any(f in path_parts for f in folders)


def main() -> int:
    ap = argparse.ArgumentParser(description="Find derived notes that reach no source.")
    ap.add_argument("vault", help="path to your notes folder (a vault of .md files)")
    ap.add_argument("--source-folders", nargs="*", default=["Sources", "Clippings"],
                    help="folders whose notes count as source material")
    ap.add_argument("--exclude-folders", nargs="*",
                    default=["Templates", "templates", "Daily", "Journal", "_system"],
                    help="folders of legitimately sourceless navigation, kept out of the report")
    ap.add_argument("--max-hops", type=int, default=2,
                    help="a note is grounded if a source is within this many outbound hops (default 2)")
    ap.add_argument("--strict", action="store_true",
                    help="shorthand for --max-hops 1: require a DIRECT link to a source")
    ap.add_argument("--link-fields", nargs="*", default=[],
                    help="frontmatter fields whose wikilink values also count as outbound links "
                         "(for vaults that cite sources in frontmatter, not the body)")
    ap.add_argument("--limit", type=int, default=25, help="how many source-orphans to print")
    ap.add_argument("--log", default=None, help="log file (default: <vault>/.grounding-log.jsonl)")
    args = ap.parse_args()

    vault = os.path.abspath(args.vault)
    if not os.path.isdir(vault):
        print(f"not a folder: {vault}", file=sys.stderr)
        return 2

    # --- 1. read every note, classify, and record outbound links -------------
    notes: dict[str, dict] = {}         # key -> {kind, rel, links:set[str]}
    by_base: dict[str, str] = {}        # basename (lowercased, no ext) -> key
    unreadable = 0
    for root, dirs, files in os.walk(vault):
        dirs[:] = [d for d in dirs if not d.startswith(".")]  # skip .obsidian, .git, caches
        for fn in files:
            if not fn.endswith(".md"):
                continue
            full = os.path.join(root, fn)
            rel = os.path.relpath(full, vault)
            key = rel
            base = os.path.splitext(fn)[0].lower()
            by_base.setdefault(base, key)
            notes[key] = {"rel": rel, "base": base, "kind": None, "links": set(), "full": full}
    for key, n in notes.items():
        try:
            text = open(n["full"], encoding="utf-8").read()
        except (UnicodeDecodeError, OSError):
            n["kind"] = "unreadable"
            unreadable += 1
            continue
        fm, body = split_frontmatter(text)
        parts = set(os.path.dirname(n["rel"]).split(os.sep))
        if has_provenance(fm) or in_any(parts, args.source_folders):
            n["kind"] = "source"
        elif in_any(parts, args.exclude_folders):
            n["kind"] = "navigation"
        else:
            n["kind"] = "derived"
        raw = set(m.group(1).strip() for m in WIKILINK.finditer(body))
        raw |= set(os.path.splitext(os.path.basename(m.group(1)))[0]
                   for m in MDLINK.finditer(body))
        for fld in args.link_fields:
            # the field's inline value, plus any list items indented beneath it
            block = re.search(rf"^{re.escape(fld)}\s*:(.*?)(?=^\S|\Z)", fm, re.M | re.S)
            if block:
                raw |= set(WIKILINK.findall(block.group(1)))
        n["links"] = raw

    # --- 2. resolve link targets to note keys --------------------------------
    def resolve(target: str) -> str | None:
        return by_base.get(target.lower()) or by_base.get(os.path.basename(target).lower())

    for n in notes.values():
        n["out"] = {r for r in (resolve(t) for t in n["links"]) if r}

    sources = {k for k, n in notes.items() if n["kind"] == "source"}
    max_hops = 1 if args.strict else max(1, args.max_hops)

    # --- 3. which notes reach a source within max_hops? ----------------------
    # Bounded reverse-graph BFS from every source. A note is grounded if a source
    # sits within max_hops outbound hops of it. One pass over the whole graph.
    rev: dict[str, list[str]] = {}
    for k, n in notes.items():
        for t in n.get("out", set()):
            rev.setdefault(t, []).append(k)
    reaches = set(sources)
    frontier = set(sources)
    for _hop in range(max_hops):
        nxt = set()
        for cur in frontier:
            for pred in rev.get(cur, ()):
                if pred not in reaches:
                    reaches.add(pred)
                    nxt.add(pred)
        if not nxt:
            break
        frontier = nxt

    derived = [k for k, n in notes.items() if n["kind"] == "derived"]
    orphans = sorted(k for k in derived if k not in reaches)
    navigation = [k for k, n in notes.items() if n["kind"] == "navigation"]

    # --- 4. report -----------------------------------------------------------
    total = len(notes)
    print(f"\ngrounding pass · {vault}")
    print(f"  {total} notes · {len(sources)} sources · {len(derived)} derived · "
          f"{len(navigation)} navigation (excluded) · {unreadable} unreadable")
    print(f"  mode: grounded if a source is within {max_hops} hop(s)"
          f"{' (strict)' if max_hops == 1 else ''}")
    print(f"\n  SOURCE-ORPHANS: {len(orphans)} derived note(s) reach no source")
    for k in orphans[:args.limit]:
        print(f"    - {k}")
    if len(orphans) > args.limit:
        print(f"    … and {len(orphans) - args.limit} more")

    # --- 5. append the count to a local log so you can watch it fall ----------
    log = args.log or os.path.join(vault, ".grounding-log.jsonl")
    row = {
        "date": _dt.date.today().isoformat(),
        "source_orphans": len(orphans),
        "derived": len(derived),
        "sources": len(sources),
        "max_hops": max_hops,
    }
    try:
        with open(log, "a", encoding="utf-8") as fh:
            fh.write(json.dumps(row) + "\n")
        print(f"\n  logged to {log}")
    except OSError as e:
        print(f"\n  could not write log ({e})", file=sys.stderr)
    return 0


if __name__ == "__main__":
    sys.exit(main())