// Simple numbering for non-book documents
#let equation-numbering = "(1)"
#let callout-numbering = "1"
#let subfloat-numbering(n-super, subfloat-idx) = {
  numbering("1a", n-super, subfloat-idx)
}

// Theorem configuration for theorion
// Simple numbering for non-book documents (no heading inheritance)
#let theorem-inherited-levels = 0

// Theorem numbering format (can be overridden by extensions for appendix support)
// This function returns the numbering pattern to use
#let theorem-numbering(loc) = "1.1"

// Default theorem render function
#let theorem-render(prefix: none, title: "", full-title: auto, body) = {
  if full-title != "" and full-title != auto and full-title != none {
    strong[#full-title.]
    h(0.5em)
  }
  body
}
// Some definitions presupposed by pandoc's typst output.
#let content-to-string(content) = {
  if content.has("text") {
    content.text
  } else if content.has("children") {
    content.children.map(content-to-string).join("")
  } else if content.has("body") {
    content-to-string(content.body)
  } else if content == [ ] {
    " "
  }
}

#let horizontalrule = line(start: (25%,0%), end: (75%,0%))

#let endnote(num, contents) = [
  #stack(dir: ltr, spacing: 3pt, super[#num], contents)
]

#show terms.item: it => block(breakable: false)[
  #text(weight: "bold")[#it.term]
  #block(inset: (left: 1.5em, top: -0.4em))[#it.description]
]

// Some quarto-specific definitions.

#show raw.where(block: true): set block(
    fill: luma(230),
    width: 100%,
    inset: 8pt,
    radius: 2pt
  )

#let block_with_new_content(old_block, new_content) = {
  let fields = old_block.fields()
  let _ = fields.remove("body")
  if fields.at("below", default: none) != none {
    // TODO: this is a hack because below is a "synthesized element"
    // according to the experts in the typst discord...
    fields.below = fields.below.abs
  }
  block.with(..fields)(new_content)
}

#let empty(v) = {
  if type(v) == str {
    // two dollar signs here because we're technically inside
    // a Pandoc template :grimace:
    v.matches(regex("^\\s*$")).at(0, default: none) != none
  } else if type(v) == content {
    if v.at("text", default: none) != none {
      return empty(v.text)
    }
    for child in v.at("children", default: ()) {
      if not empty(child) {
        return false
      }
    }
    return true
  }

}

// Subfloats
// This is a technique that we adapted from https://github.com/tingerrr/subpar/
#let quartosubfloatcounter = counter("quartosubfloatcounter")

#let quarto_super(
  kind: str,
  caption: none,
  label: none,
  supplement: str,
  position: none,
  subcapnumbering: "(a)",
  body,
) = {
  context {
    let figcounter = counter(figure.where(kind: kind))
    let n-super = figcounter.get().first() + 1
    set figure.caption(position: position)
    [#figure(
      kind: kind,
      supplement: supplement,
      caption: caption,
      {
        show figure.where(kind: kind): set figure(numbering: _ => {
          let subfloat-idx = quartosubfloatcounter.get().first() + 1
          subfloat-numbering(n-super, subfloat-idx)
        })
        show figure.where(kind: kind): set figure.caption(position: position)

        show figure: it => {
          let num = numbering(subcapnumbering, n-super, quartosubfloatcounter.get().first() + 1)
          show figure.caption: it => block({
            num.slice(2) // I don't understand why the numbering contains output that it really shouldn't, but this fixes it shrug?
            [ ]
            it.body
          })

          quartosubfloatcounter.step()
          it
          counter(figure.where(kind: it.kind)).update(n => n - 1)
        }

        quartosubfloatcounter.update(0)
        body
      }
    )#label]
  }
}

// callout rendering
// this is a figure show rule because callouts are crossreferenceable
#show figure: it => {
  if type(it.kind) != str {
    return it
  }
  let kind_match = it.kind.matches(regex("^quarto-callout-(.*)")).at(0, default: none)
  if kind_match == none {
    return it
  }
  let kind = kind_match.captures.at(0, default: "other")
  kind = upper(kind.first()) + kind.slice(1)
  // now we pull apart the callout and reassemble it with the crossref name and counter

  // when we cleanup pandoc's emitted code to avoid spaces this will have to change
  let old_callout = it.body.children.at(1).body.children.at(1)
  let old_title_block = old_callout.body.children.at(0)
  let children = old_title_block.body.body.children
  let old_title = if children.len() == 1 {
    children.at(0)  // no icon: title at index 0
  } else {
    children.at(1)  // with icon: title at index 1
  }

  // TODO use custom separator if available
  // Use the figure's counter display which handles chapter-based numbering
  // (when numbering is a function that includes the heading counter)
  let callout_num = it.counter.display(it.numbering)
  let new_title = if empty(old_title) {
    [#kind #callout_num]
  } else {
    [#kind #callout_num: #old_title]
  }

  let new_title_block = block_with_new_content(
    old_title_block,
    block_with_new_content(
      old_title_block.body,
      if children.len() == 1 {
        new_title  // no icon: just the title
      } else {
        children.at(0) + new_title  // with icon: preserve icon block + new title
      }))

  align(left, block_with_new_content(old_callout,
    block(below: 0pt, new_title_block) +
    old_callout.body.children.at(1)))
}

// 2023-10-09: #fa-icon("fa-info") is not working, so we'll eval "#fa-info()" instead
#let callout(body: [], title: "Callout", background_color: rgb("#dddddd"), icon: none, icon_color: black, body_background_color: white) = {
  block(
    breakable: false, 
    fill: background_color, 
    stroke: (paint: icon_color, thickness: 0.5pt, cap: "round"), 
    width: 100%, 
    radius: 2pt,
    block(
      inset: 1pt,
      width: 100%, 
      below: 0pt, 
      block(
        fill: background_color,
        width: 100%,
        inset: 8pt)[#if icon != none [#text(icon_color, weight: 900)[#icon] ]#title]) +
      if(body != []){
        block(
          inset: 1pt, 
          width: 100%, 
          block(fill: body_background_color, width: 100%, inset: 8pt, body))
      }
    )
}



// arxivai — Quarto Typst template partial.
// Replaces Quarto's default `article()` with the arxivai preprint layout.
// Both packages are importable from raw ```{=typst}``` blocks in the body:
//   #arxivai.operation-box(...)   #arxivai.app-figure(...)   #lettreai.badge(...)

#import "lettreai.typ" as lettreai
#import "arxivai.typ" as arxivai

// ---------------------------------------------------------------------------
// Re-theme Quarto's automatic callouts (same contract as lettreai: block
// nesting kept structurally identical to Quarto's default so the crossref
// figure show rule in definitions.typ keeps working).
// ---------------------------------------------------------------------------
#let callout(
  body: [],
  title: "Callout",
  background_color: rgb("#dddddd"),
  icon: none,
  icon_color: black,
  body_background_color: white,
) = {
  let accent = if icon_color == black { lettreai.colors.oxford-blue } else { icon_color }
  block(
    breakable: false,
    fill: lettreai.tint(accent, 0.94),
    stroke: (left: 3pt + accent, rest: 0.5pt + lettreai.tint(accent, 0.5)),
    width: 100%,
    radius: 3pt,
    block(
      inset: 1pt,
      width: 100%,
      below: 0pt,
      block(
        fill: lettreai.tint(accent, 0.84),
        width: 100%,
        inset: 8pt,
      )[#if icon != none [#text(accent, weight: 900)[#icon] ]#text(fill: lettreai.shade(accent, 0.25), weight: 700)[#title]]) +
      if (body != []) {
        block(
          inset: 1pt,
          width: 100%,
          block(fill: body_background_color, width: 100%, inset: 8pt, body))
      }
    )
}

// ---------------------------------------------------------------------------
// arXiv preprint article — same signature contract as Quarto's default,
// extended with keywords and running-title.
// ---------------------------------------------------------------------------
#let article(
  title: none,
  subtitle: none,
  authors: none,
  keywords: (),
  running-title: none,
  book: false,
  date: none,
  abstract-title: none,
  abstract: none,
  thanks: none,
  cols: 1,
  lang: "en",
  region: "US",
  font: none,
  fontsize: 10.5pt,
  title-size: 1.7em,
  subtitle-size: 1.15em,
  heading-family: none,
  heading-weight: "bold",
  heading-style: "normal",
  heading-color: black,
  heading-line-height: 0.65em,
  mathfont: none,
  codefont: none,
  linestretch: 1,
  sectionnumbering: none,
  linkcolor: none,
  citecolor: none,
  filecolor: none,
  toc: false,
  toc_title: none,
  toc_depth: none,
  toc_indent: 1.5em,
  doc,
) = {
  // PDF metadata.
  set document(title: title)
  set document(
    author: authors.map(a => lettreai.to-string(a.name)).join(", ", last: " & "),
  ) if authors != none and authors != ()
  set document(
    keywords: keywords.map(k => lettreai.to-string(k)),
  ) if keywords != none and keywords != ()

  set par(justify: true, leading: linestretch * 0.65em)
  set text(lang: lang, region: region, size: fontsize)
  show math.equation: set text(font: mathfont) if mathfont != none

  set heading(numbering: sectionnumbering)

  // Running header + footer; book mode gets chapter-aware heads and the
  // O'Reilly outer-margin geometry.
  let rt = if running-title != none { running-title } else if title != none { title } else { [] }
  set page(
    header: if book { arxivai.book-header() } else { arxivai.arxiv-header(rt) },
    footer: if book { arxivai.book-footer() } else { arxivai.arxiv-footer() },
  )
  set page(margin: arxivai.book-margin) if book

  // Resolve fonts: explicit Quarto fonts win, else theme defaults.
  let resolved-font = if font != none { font } else { lettreai.body-font }
  let resolved-code = if codefont != none { codefont } else { lettreai.mono-font }
  let resolved-heading-color = if heading-color != black { heading-color } else { lettreai.colors.oxford-blue }
  let resolved-link = if linkcolor != none { rgb(lettreai.to-string(linkcolor)) } else { lettreai.colors.oxford-blue }

  // Paint everything with the shared theme, then the figure rules.
  show: lettreai.theme-rules.with(
    fonts: resolved-font,
    code-font: resolved-code,
    heading-color: resolved-heading-color,
    link-color: resolved-link,
  )
  show: arxivai.arxiv-figure-rules
  // Book chapter rule is the innermost show, so it transforms level-1
  // headings FIRST (inner rules apply first), replacing them with chapter
  // openers before the theme's h1 rule could wrap them in a container.
  // Synthetic non-outlined headings (e.g. the outline's own title) are
  // excluded inside book-heading-rules.
  show: doc2 => if book { arxivai.book-heading-rules(doc2) } else { doc2 }

  show ref: set text(fill: rgb(lettreai.to-string(citecolor))) if citecolor != none

  // Preprint front matter.
  arxivai.arxiv-title-block(
    title: title,
    subtitle: subtitle,
    authors: authors,
    date: date,
  )
  if abstract != none {
    arxivai.arxiv-abstract(
      abstract,
      abstract-title: if abstract-title == none { [Abstract] } else { abstract-title },
    )
  }
  arxivai.arxiv-keywords(keywords)

  // Book mode: the front matter is a title page of its own.
  if book { pagebreak(weak: true) }

  // Optional table of contents (its own page in book mode).
  if toc {
    block(above: 0.5em, below: 2em, lettreai.lettreai-outline(
      title: if toc_title == none { auto } else { toc_title },
      depth: toc_depth,
      indent: toc_indent,
    ))
    if book { pagebreak(weak: true) }
  }

  if cols > 1 { columns(cols, doc) } else { doc }
}
#let brand-color = (:)
#let brand-color-background = (:)
#let brand-logo = (:)

#set page(
  paper: "a4",
  margin: (x: 2.4cm,y: 2.6cm,),
  numbering: "1",
  columns: 1,
)

#show: doc => article(
  title: [PDAI: A Certified Document Infrastructure for Computational Historical Scholarship],
  subtitle: [From PDF to a Queryable, Audited, and Learning-Enabled Corpus],
  authors: (
    ( name: [Gerd Graßhoff],
      affiliation: [Humboldt-Universität zu Berlin, Deutsches Museum],
      email: [gerd.grasshoff\@hu-berlin.de],
      orcid: [0000-0002-4450-357X] ),
    ),
  keywords: ([digital humanities],[document processing],[optical character recognition],[computational epistemology],[PDAI format],[schema-first OCR],[scholarly infrastructure],[large language models],[grounding evidence],),
  running-title: [PDAI Document Infrastructure],
  abstract: [Computational epistemology --- the systematic study of how knowledge claims in historical science can be evaluated computationally --- requires primary sources that are queryable down to the page and sentence. This paper presents PDAI (Portable Document AI), a certified document infrastructure that transforms scanned scholarly editions into structured, queryable corpora with full production audit and learning-enabled quality assurance. The infrastructure has three layers: the PDAI format (a flat, predictable XML directory structure validated against a declared DTD grammar), the production pipeline (a Partitur-governed sequence from ingest through schema-first OCR, pilot-gated batch processing, and a mandatory certify-and-learn stage), and the contextual data stores (corpus file with lemma postings and BM25, famulus file, gazetteer, LWgrammar lexicon, register of individuals) that make a certified page answerable to structured scholarly queries. A reference layer, the Famulus Reference, gives every document, page and segment a stable `fr:` address, a citation drawn only from recorded bibliographic data, and provenance and validation records, kept in one DuckLake and served by one MCP server. The paper describes the nine structural categories of the PDAI page grammar and shows through five case studies how they capture the genuine typographic diversity of early modern scholarly editions --- epistolary apparatus, astronomical ephemerides, cuneiform transliteration, manuscript catalogue entries, and philosophical correspondence. A page enters the corpus only after the audit gate: every defect is attributed to one of four root-cause layers, registered as a versioned learning case, and closed through an A/B-gated instruction improvement loop. The result is a certified chain from PDF to the ai-viewer endpoint where pages serve as Grounding Evidence for computational epistemological inquiry --- citable, provenance-tracked, and regression-anchored.

],
  abstract-title: "Abstract",
  font: ("Source Sans 3",),
  sectionnumbering: "1.1",
  toc: true,
  toc_title: [Table of contents],
  toc_depth: 3,
  doc,
)

= Introduction: Documents as Grounding Evidence
<introduction-documents-as-grounding-evidence>
The history of science is a record of claims. Kepler claims that planetary motion obeys an area law; Euler derives the shape of tidal forcing from a Newtonian potential; the Babylonian astronomical diaries record Jupiter's daily position in degrees and seconds. These claims are embedded in primary sources --- letters, treatises, observation records --- most of which reached the digital age as scanned PDFs: visually faithful, computationally opaque.

#arxivai.sidebar(title: [Grounding Evidence])[
  *Grounding Evidence* is the direct textual or observational basis on which
  a historical knowledge claim rests — as distinct from secondary literature
  that interprets or summarises it. A page of Kepler's correspondence is
  Grounding Evidence for claims about his reasoning; a monograph about Kepler
  is Sekundärevidenz. The distinction is operational: PDAI pages are structured
  to serve as Grounding Evidence, which means they must be citable at the
  sentence level, provenance-tracked, and answerable to structured queries
  without paraphrase.
]
A scanned PDF preserves what a document looks like. It cannot answer the question: #emph[on how many pages of the Kepler correspondence does Kepler mention the law of refraction?] To answer this question computationally, a system needs not images but structured transcriptions --- transcriptions that record not just characters but the elements of scholarly apparatus: headers, sections, paragraphs, marginal notes, footnotes, figures, tables, mathematical expressions, and the epistolary skeleton of a letter. It needs those transcriptions validated against a declared grammar so that queries can be written against a stable schema. And it needs provenance: which model produced this transcription, under which instruction, on which date.

The Portable Document AI (PDAI) infrastructure addresses these requirements. This paper describes its format, its production pipeline, and the certify-and-learn system that makes a PDAI page a trustworthy unit of Grounding Evidence.

Three companion papers carry related material: the orchestration architecture behind the query pipeline is presented in @grasshoff2026aipartitur\; the linguistic index layer (lemma tables, dependency parses, DuckDB query engine) in @grasshoff2026aistanza\; and the research platform that reads PDAI pages as evidence for scholarly operations in @grasshoff2026aierasmus. The reference layer is documented with its package (`pdai/famulus-reference/README.md`); its addresses are consumed by the justification library famulus-proof (`development/famulus-proof`).

= The PDAI Format
<the-pdai-format>
== The problem with PDF
<the-problem-with-pdf>
The PDF standard @pdf2020 solves a real problem: document appearance is preserved across platforms. This made it the canonical format for digitised historical collections. But the solution creates a different problem for computational scholarship.

A scanned PDF encodes position on a page, not structure. A footnote marker in the text and its referent at the page foot share no programmatic connection; a column in a table of astronomical data has no type. Even PDFs with embedded text layers from conventional OCR contain errors, lose structure, and cannot distinguish a marginal annotation from the main text. The consequence is that the scholarly question --- what does this document say about this topic --- can be asked of a PDF only through full-text string search, which neither understands structure nor tolerates the orthographic variation of historical texts.

The TEI guidelines @tei2023 offer rich semantic encoding, but at a cost: TEI markup is written by human editors and is not automatically produced from scans. For large historical corpora --- tens of thousands of pages across dozens of editions --- TEI encoding is not feasible. What is needed is automatic production of structured XML from page images, at sufficient quality to serve as Grounding Evidence.

== The PDAI directory structure
<the-pdai-directory-structure>
A PDAI document is a directory on the filesystem. The directory structure separates concerns while maintaining clear relationships between layers:

```
/Volumes/PDAI-1/pdai-collections/<collection>/
  status.json                 readiness manifest (pdai-complete)
  <collection>_corpus.duckdb  sentence corpus, lemma table, BM25 (corpus/2)
  <collection>_famulus.duckdb derived tables: FQL, evidence, sentence records,
                              dossier, famulus reference, letters, register
  gazetteer_*.parquet         person, place and domain gazetteer
  _lake/                      the collection's Parquet files in the DuckLake
  <biblabel>/
    <biblabel>.pdf            original source PDF
    biblio.xml                bibliographic identity (biblio.dtd)
    pdai-config.json          harvest and processing record
    pages/
      page_NNN/
        page_NNN.png          page image at production resolution
        page_NNN.txt          raw OCR text (fastimage, used as signal only)
        page_NNN.xml          structured PDAI XML (the primary output)
    segments/
      letters/                materialised letter records
    structure.xml             document-level assembly

~/ai-mac-studio/pdai/collections/<collection>/quality/
  <vol>_defects.json          structural audit results
  <vol>/
    tag_hierarchy_protocol.jsonl   structural fingerprints for regression
    learning_fault_dossier.json    fault cases for the learning loop
    audit_report_<vol>.pdf         LettreAI Typst certification report
```

The separation is principled. Source images occupy `pages/`\; OCR outputs are co-located with the page. `segments/` holds assembled records that span multiple pages. `structure.xml` is the document-level view. The audit and learning layer `quality/` lives with the collection's code, not with its data: it accumulates without touching production files.

This structure embodies two design principles: separation of concerns (no processing step overwrites source data) and accumulative growth (the quality folder grows as auditing deepens, without invalidating transcriptions).

== The canonical page DTD
<the-canonical-page-dtd>
Every PDAI page is validated against `page.dtd` v4.0 (or a collection wrapper of it), maintained at `pdai/pipeline/ai-scholar/src/ai_scholar/schemas/page.dtd`. The DTD is the grammar contract: any query written against the PDAI schema can rely on the elements in #ref(<tbl-dtd-categories>, supplement: [Table]) being present and structurally correct.

The page language, together with the collection wrappers described below, covers nine structural categories. Seven are declared by the canonical grammar itself; the epistolary apparatus comes from the letter wrapper `page_letter.dtd`, and the `<equation>` element from the wrappers that need it:

#figure([
#table(
  columns: (16.22%, 48.65%, 35.14%),
  align: (auto,auto,auto,),
  table.header([Category], [Elements], [Role in scholarly apparatus],),
  table.hline(),
  [Page frame], [`<page pidx="…">`, `<meta>`, `<header>`, `<footer>`], [Page address and processing provenance],
  [Running text], [`<p>`, `<parno>`, `<br>`], [Prose, numbered paragraphs, verse and list lines],
  [Typography], [`<hi rend="…">`], [Emphasis (italic, bold, smallcaps, superscript, subscript, spaced)],
  [Epistolary apparatus], [`<letter-part>`, `<letter-head>`, `<sender>`, `<addressee>`, `<dateline>`, `<date>`, `<place>`, `<archive>`, `<salutation>`, `<letter-closing>`, `<valediction>`, `<signature>`, `<address>`], [Letter structure and archival provenance (`page_letter.dtd`)],
  [Mathematics], [LaTeX `$…$` and `$$…$$` in the text; `<equation>` in wrappers], [Inline and display expressions],
  [Figures], [`<figure ref="…">` (with `fr`, `src`, `bbox`, `kind`), `<caption>`, `<description>`], [Illustrations with transcribed caption and a model-made description],
  [Tables], [`<table>`, `<tr>`, `<th>`, `<td>`], [Tabular data (never flattened to `<p>`)],
  [Marginalia], [`<marginal side="…" type="…">`], [Everything printed in a margin; markers line-number, folio, symbol; notes topic, reference, date],
  [Footnotes], [`<footnotes>`, `<footnote>`, `<fn>`], [Page-bottom apparatus with inline markers],
)
], caption: figure.caption(
position: top, 
[
The nine structural categories of the PDAI page language and its wrappers. Elements a collection adds for its own printed conventions are listed in the next table.
]), 
kind: "quarto-float-tbl", 
supplement: "Table", 
)
<tbl-dtd-categories>


The `<meta>` element records the provenance of each page:

```
<meta model="qwen3.8:27b-mlx" engine="qwen3.8:27b-mlx"
      instruction_template="kepler-ocr"
      partitur="kepler-ocr@3f400c0df6fa"
      layers="page-language,base,qwen38"
      overlay="kepler-ocr.qwen38.j2"
      lang="orig"
      processed_date="2026-09-28"
      output_schema="kepler-prim/page.dtd v2.0" />
```

This provenance chain is unbroken: every page records which model and which composed instruction produced it. `partitur` is the instruction family and the first twelve hex digits of the SHA-256 of the composed prompt, `layers` the layers it was composed from (below). A scholar who doubts a transcription can reproduce it with identical configuration. Detecting when reprocessing is warranted --- because a model was updated or an instruction improved --- is a query over the `meta` element's attributes. The reference layer records the page as well: one `fr_provenance` row per page names the page XML and its SHA-256, and one row per document names the OCR model of `pdai-config.json`\; the per-page model and partitur stay in `<meta>` and are not yet parsed into `fr_provenance`.

== The page language: one core, collection wrappers
<the-page-language-one-core-collection-wrappers>
Every PDAI grammar includes one core module, `pdai-core.mod`, which declares what all collections share: the margin, the processing record `<meta>`, figures and the renditions of `<hi>`\; tables come from `pdai-table.mod`. The canonical `page.dtd` (v4.0) writes each of its content models as a parameter entity. A collection grammar in `config/page.dtd` is therefore a #emph[wrapper]: it sets the entities it needs, declares only the elements the canonical page lacks, and includes `page.dtd`. The letter page grammar `page_letter.dtd` (v4.3) is such a wrapper too.

#arxivai.method-box(title: [The extension rule])[
  A collection grammar adds to the page language. It never removes or renames a
  core element or margin type, and it does not introduce record-level assembly
  into the page grammar. The page grammar records what is printed on this
  physical page; the record grammar (letter.dtd, catalogue.dtd, tablet.dtd)
  assembles the logical record from multiple pages.
]
#table(
  columns: (26.09%, 73.91%),
  align: (auto,auto,),
  table.header([Collection], [Its own elements beyond the page language],),
  table.hline(),
  [`kepler-prim`], [letter head and closing families, `equation`, `lineno`, `fnmark`\; KGW 19: `document-entry` and margin type `doc-id`],
  [`copernicus-prim`], [critical apparatus (`apparatus`, `app`), table of contents (`toc`, `toc-entry`)],
  [`diaries`], [position-ordered metadata `block`s with `field`, `witness`, `coverage`, `comment`, `row`\; typed `section`],
  [`medieval-sec`], [catalogue `entryblock` with its fields, `register`\; margin type `entry-number`],
  [letter collections], [`page_letter.dtd` wrappers: `letter-part`, head, body, closing, enclosure and apparatus families],
)
#strong[Margins.] Everything printed in a margin is a `<marginal side="…" type="…">` standing directly before the first word of the printed line it is printed beside. The type is one word of a closed list declared in the core: the markers `line-number`, `folio`, `symbol` and the notes `topic`, `reference`, `date`. A collection may add a type only through the core's extension entity. The same list is read from the grammar by the OCR prompt, the normalizer (which maps the aliases `note`, `marginal-note`, `label`, `gloss` to `topic`), the audit (`SCH-MARGINAL-TYPE`) and the viewer, so the four cannot disagree.

#strong[Prompts.] An OCR prompt is composed in fixed layers: the page-language fragment, rendered with the vocabulary of the collection's grammar; the collection base; an optional volume layer; the engine overlay, which holds only corrections measured for that engine. The composed prompt is the #emph[partitur] stamped on each page (`<meta partitur="…" layers="…">`).

The flattened grammars are deployed with the collection; a census of 28 September 2026 over all 159,170 page files on the production volume found 93,279 valid before and 95,751 after the restructuring, with no document losing a valid page.

== PDAI compared with alternative formats
<pdai-compared-with-alternative-formats>
#table(
  columns: (31.03%, 17.24%, 31.03%, 20.69%),
  align: (auto,auto,auto,auto,),
  table.header([Feature], [PDF], [TEI-XML], [PDAI],),
  table.hline(),
  [Visual fidelity], [Excellent], [Variable], [Good (via images)],
  [Semantic structure], [Minimal], [Excellent], [Excellent],
  [Automatic production from scans], [No], [No], [Yes],
  [AI processable], [Poor], [Moderate], [Excellent],
  [Progressive enhancement], [No], [Manual only], [Automated agents],
  [Provenance tracking], [None], [Manual], [Automatic per page],
  [Schema validation], [No], [DTD/Schema], [DTD throughout],
  [Interactive scholarly viewer], [Reader apps], [Custom], [Built-in],
  [Item-level citation], [No], [Manual], [`fr:` address with citation export],
)
PDAI does not replace TEI for critical editions requiring diplomatic accuracy at the level of individual manuscript readings; nor does it replace PDF for visual reproduction. Its position is the gap between the two: automatic production at scale, with sufficient structural precision for computational scholarly inquiry.

= Five Document Types: The Expressiveness of the Grammar
<five-document-types-the-expressiveness-of-the-grammar>
The nine structural categories of the PDAI grammar are not abstractions: they are derived from the actual typographic practice of historical scholarly editions. This section shows the grammar at work across five document types that represent the range of PDAI collections.

== Euler's Tidal Theory (E57): mathematical Latin treatise
<eulers-tidal-theory-e57-mathematical-latin-treatise>
Leonhard Euler's #emph[Inquisitio Physica in Causam Fluxus ac Refluxus Maris] (1740, Euler Archive E57) is a 105-page Latin mathematical treatise in 8 chapters with 130 consecutively numbered paragraphs @euler1740.

A page of the transcription in the `enestroem` collection shows three structural categories: the running header, running text with a numbered paragraph (`<parno>`), and mathematics written as LaTeX inside the text (page 13, abridged; the `<meta>` record is omitted):

```
<page pidx="013">
  <header>32 INQUISITIO PHYSICA [252-253]</header>
  <main>
    <figure ref="fig_013_1">
      <p>Fig. 2</p>
    </figure>
    <p>Solem Lunamve in $S$, existente distantia $SC = a$; … </p>
    <parno>21.</parno> Cum igitur superficies sphaerica perinde ad Solem
       sive Lunam in $S$ sollicitetur, …
  </main>
</page>
```

The page also shows what the audit is for. Its figure carries neither a `<caption>` nor a `<description>`: the printed label "Fig. 2" was read as a paragraph, and the parno stands outside a `<p>`. The figure census (see § 5.3) counts such a figure as `empty`, and the DTD validation rejects the misplaced elements. A `<description>`, where present, is not a transcription of printed text but a model-made description of the diagram, written to enable query and display; it is never page text, never a ground of a justification and never the title of a citation.

== Kepler's Correspondence (KGW 13--18): epistolary apparatus
<keplers-correspondence-kgw-1318-epistolary-apparatus>
The Kepler Gesammelte Werke correspondence volumes (KGW 13 through 18) contain Kepler's letters up to 1630 @kgw13. Their structure is epistolary: formal headers, archival references, letter bodies, and closing signatures. The `page_letter.dtd` wrapper captures this (KGW 13, printed page 24, abridged):

```
<page pidx="page_038" volume="kgw-13" printed-page="24"
      output_schema="page_letter.dtd v4.0">
  <meta model="qwen3.8:27b-mlx" instruction_template="kepler-ocr"
        lang="orig" processed_date="2026-08-20" … />
  <header>24 1595 Nr. 19</header>
  <main>
    <letter-part nr="19"><letter-head>
      <reference>19. KONRAD VISCHSES AN KEPLER IN GRAZ</reference>
      <date>14./24. Mai 1595</date>
      <archive>Pulkowo, Sternwarte, Kepler-Mss. Bd. XIX,
        Bl. 23–24. Eigenhändig</archive>
    </letter-head>
    <p><marginal side="right" type="line-number">25</marginal>Vitam et
       Linguam precor, aut linguae vicariam dexteram. …</p>
    …
```

The epistolary apparatus --- `<reference>`, `<date>`, `<archive>`, and where the edition prints them `<sender>` and `<addressee>` --- enables queries that the text alone does not support: #emph[in how many letters between 1610 and 1615 does Kepler discuss refraction?] This requires structured access to both the letter header (for the date filter) and the letter body (for the subject). The letter register of the collection (`kepler-prim/register/letters.parquet`) holds 934 letters of KGW 13--18; assembled letter records (`segments/letters/`) exist for KGW 13--16, 631 in all.

== Astronomical Diaries (ADRT 1--7): cuneiform transliteration
<astronomical-diaries-adrt-17-cuneiform-transliteration>
The #emph[Astronomical Diaries and Related Texts from Babylonia] \[bibliographic entry to be supplied\] are a continuous record of celestial and terrestrial observations spanning 750--60 BCE. The PDAI `diaries` collection covers volumes I through VII with 3,853 transcribed pages (3,546 text pages and 307 plate pages of volumes I--III). Their format is not epistolary but tabular-transliteration: Sachs and Hunger's edition prints the transliteration of cuneiform tablets alongside the translation and scholarly commentary.

The `diaries` grammar adds position-ordered metadata blocks and typed sections to the page language; a translation page of ADRT 1 reads (abridged):

```
<page pidx="042">
  <meta model="qwen3.8:27b-mlx" instruction_template="diaries_translit_ocr"
        lang="orig" processed_date="2026-08-15" page_index="042"/>
  <header>No. -651 45</header>
  <main>
    …
    <section type="lunar">
      <p style="tr"><parno>4'</parno>The north wind blew. Lunar eclipse …</p>
      <p style="tr"><parno>5'</parno>The 20th<hi rend="superscript">?</hi>, the
         moon was surrounded by a halo, (and) Mercury [stood] in i[t ....]</p>
    </section>
    <section type="planetary">
      <p style="tr"><parno>7'</parno>.... The 5th, Mercury's first appearance
         in the east in Pisces. [....]</p>
    </section>
    …
```

The tablet's line numbers (`<parno>4'</parno>`) stay with the line they label, and the lines are grouped into typed sections (`<section type="…">`: lunar, planetary, weather, river, historical) assigned during OCR. A query for Mercury's first appearances can therefore be restricted to planetary sections without reading the commentary; the section type is a reading of the transcriber, not a mark printed in the edition.

== Bischoff's Handschriften-Katalog (medieval-sec): catalogue entries
<bischoffs-handschriften-katalog-medieval-sec-catalogue-entries>
Bernhard Bischoff's #emph[Katalog der festländischen Handschriften des neunten Jahrhunderts] (3 volumes, 1,641 pages) is a dense scholarly catalogue: each entry identifies a Latin manuscript, its present location, foliation, script type, and Bischoff's attribution. The entry structure extends the page language with the catalogue's own `<entryblock>` and its fields (`<shelfmark>`, `<foliation>`, `<contents>`, `<script>`, `<literature>`, `<origin>` …).

A single catalogue entry spans between one and several pages; the page grammar records the entry block, and `catalogue.dtd` assembles the complete entry:

```
<page pidx="203">
  <meta model="qwen3.8:27b-mlx" instruction_template="medieval-sec-catalogue-ocr"
        lang="de" processed_date="2026-08-19" page_index="203"
        output_schema="medieval-sec/page.dtd v1.0" … />
  <header place="METZ" folio="189">METZ    189</header>
  <main>
    <entryblock position="1" type="main" state="partial"
                truncated="top"> … </entryblock>
    <entryblock position="2" type="main" state="complete">
      <number value="2781">2781</number>
      <shelfmark>Ms. 732</shelfmark>
      <foliation scope="true">(Bl. 4)</foliation>
      <contents><title>Vetus Testamentum</title> (II Par 29,11–27).</contents>
      …
      <cla>CLA VI.790.</cla>
      <origin place="Metz" date="VIII./IX. Jh.">– Metz, VIII./IX. Jh. –</origin>
    </entryblock>
    …
```

The page header carries the library place, the entry its `<shelfmark>`, and the attributes of `<origin>` the place and date of writing that Bischoff assigns. A question such as #emph[how many manuscripts does Bischoff attribute to Regensburg in the second half of the ninth century?] is therefore a query over `<origin place="…" date="…">`, not a string search over the running text. An entry cut by the page break is marked `state="partial"`, so the assembly across pages is visible in the page grammar itself.

== The Euler-Goldbach Correspondence: segment-level assembly
<the-euler-goldbach-correspondence-segment-level-assembly>
The correspondence between Euler and Christian Goldbach (1729--1764) comprises 196 letters spanning the development of number theory, analysis, and mathematical physics. The collection `eulerOO-IVA` holds it as volume IVA 4 of the Opera omnia (document `978-3-0348-0892-7`, 1,256 pages; bibliographic entry to be supplied). This edition demonstrates segment-level processing: letters span multiple pages, and the query layer must be able to treat a complete letter as a unit.

The letters of this volume are segmented by the `segment` stage of the FQL sidecar builder, which writes one row per letter, with sender, recipient, date, the edition's regest and the page range of the original and of the translation, into the FQL tables `fq_letters` and `fq_letter_pages` of `eulerOO-IVA_famulus.duckdb` (formerly the separate `eulerOO-IVA_fql.duckdb`). The record of letter 51 reads (abridged):

```
{
  "letter_id": "IVA4:51",
  "sender": "C. Goldbach",
  "recipient": "L. Euler",
  "date_when": "1742-06-07",
  "date_text": "(May 27th) June 7th, 1742",
  "register_nr": 765,
  "pidx_from": 197,
  "pidx_to": 199,
  "tags": ["goldbach's appointment", "pension", "number theory",
           "goldbach's conjecture"]
}
```

Letter 51, Goldbach's letter to Euler of 7 June 1742, is the letter in which Goldbach states the conjecture that bears his name; its record carries the tag `goldbach's conjecture`. The original occupies pages 197--199 of the volume (`pidx`), the translation pages 710--714. A query for #emph[letters in which Goldbach discusses prime sums] selects this letter from the 196 by its record and then reads its sentences, without requiring a model to read the whole volume.

= The Production Pipeline
<the-production-pipeline>
== Architecture overview
<architecture-overview>
The PDAI production pipeline is a Partitur-governed sequence of seven stages. Its OCR part is formalised as the Partitur `pilot-gated-ocr-pipeline` v1.2.0, registered at goal path `pipeline/ocr/pilot-gated` in the central Partitur register; its eleven stages run from the assessment of the document's scale through profile, per-document grammar, pilot sample, pilot OCR, structural gate, layer training, fit check and measured model selection to the full run and the certify-and-learn stage. The Partitur is a plan with declared free inputs, of which `collection` and `volume` are required and the others (`primary_model`, `dtd_path`, sample size and further options) have defaults; concrete values are supplied through a setting file before execution. The Conductor never executes an unbound Partitur.

#block[

#block[
#box(image("ai-scholar-grasshoff_files/figure-typst/mermaid-figure-1.png", height: 5.6in, width: 4.12in))

]

]
Each arrow is a concrete command or service call. Each node is a system component with defined inputs, outputs, and failure modes.

A collection that does not yet exist enters through the intake skill `pdai-collection-intake` (Partitur goal path `pipeline/intake/collection`), which wraps the same stages around an import folder in eight steps: inventory, profile, DTD draft, segment plan, a review gate that requires the owner's recorded approval and a pilot with zero blockers, build, famulus DB and lake export. The reference case is the `hartmann` collection, whose grammar the owner approved on 17 October 2026; `karman-context` followed on 19 October (`docs/dtd_review.md` of each project).

== Stage 1: Ingest and classify
<stage-1-ingest-and-classify>
The source PDF or scanned image folder is ingested by `ai-classifypdf ingest`. Page images are extracted at production resolution (typically 300 DPI for printed text, 400 DPI for manuscripts). The document is classified by its material, one of nine classes of the document-class table (`pdai/skills/pipeline/document-profile/document_classes.yaml`), or `unclassified` when none fits. A class fixes the OCR template, the page and record DTD, the Partitur and the stylesheet:

- printed scholarship --- journals, monographs, editions of correspondence and early modern prints (#emph[printed-scholarly-journal], #emph[printed-scholarly-monograph], #emph[historical-correspondence-edition], #emph[early-modern-print])
- special scripts (#emph[fraktur-print], #emph[transliteration-corpus]) --- Fraktur and special scripts pass the script gate in `ai-scholar` before any production OCR
- archival material (#emph[handwritten-archival-correspondence], #emph[typescript-archival-record]) --- mixed archival files are classified page by page with `gemini-3.8-flash`
- formalised catalogues (#emph[formalised-catalogue]) --- structured entry format, escalated to the schema-first derivation

All classes name the same OCR model, `kimi-k2.7-code:cloud`. Classification produces `ocr_classification.json` and `collection_profile.yaml`, which the OCR stage reads automatically. Classification is #strong[always before OCR]\; no production OCR runs without a completed `ocr_classification.json`.

== Stage 2: Schema derivation
<stage-2-schema-derivation>
For formalised editions, the schema derivation reads the printed front matter of the volume and derives the record schema before any OCR. Every schema element is graded by evidence:

- `front-matter`: cited from the title page or the edition's introduction --- authoritative
- `facsimile-only`: observed in page images but not described in the front matter --- provisional
- `unattested`: hypothesised but not observed --- must not be included

The unit of derivation is the document. For each document the derivation writes, under `config/volumes/<doc>/`, a wrapper `page.dtd` of the canonical grammar, one narrower wrapper per page class where needed, and the record schema with its evidence grades, together with the audit rules `config/audit_rules_<doc>.yaml`\; the collection's `config/page.css` sets the display. These files constitute the schema package. No production OCR runs without a complete schema package.

#arxivai.method-box(title: [Schema before OCR])[
  The schema is derived from the printed document *before* OCR is run.
  An element found in the printed front matter is treated as authoritative
  and included; an element observed in the facsimile but not described is
  provisional; an element neither described nor observed must not be
  invented. This discipline prevents schema drift: the OCR model is not
  free to introduce structural categories that the edition's editor did not
  declare.
]
== Stage 3: The pilot gate
<stage-3-the-pilot-gate>
For any document of more than 50 pages, the pilot gate is mandatory before batch OCR is unlocked. Five pages are drawn as a reproducible random sample spanning the beginning, middle, and end of the document, with diversity across layout types: running text, headers, tables, figures, and indexes.

The pilot sample is audited in three independent passes:

+ #strong[DTD validation]: every pilot page must be well-formed XML and valid against `page.dtd`. Zero structural blockers are required.
+ #strong[Declarative rule conformance]: `audit_rules.yaml` checks are applied --- vocabulary sets, table cue rules, forbidden tokens. Zero blockers required.
+ #strong[Visual facsimile comparison]: the rendered page in `ai-viewer` is compared against the facsimile scan. VIS-\* defects (table flattened to prose, marginal gutter overflow, hyphenated word split across two `<p>` elements) are recorded and attributed to one of the four root-cause layers.

If any blocker is raised, the improvement loop runs before batch unlock: the Jinja2 prompt overlay is revised, the normalizer regex is updated, or `page.css` is calibrated. The sample is re-audited until 100% PASS (0 blockers).

== Stage 4: Schema-first OCR batch
<stage-4-schema-first-ocr-batch>
The full volume is processed by the schema-first runner with the engine that the Partitur's model-selection stage chose by measured evidence. The default OCR engine for printed and handwritten material is `kimi-k2.7-code:cloud`, run through Ollama Cloud (owner decision of 26 September 2026, which retired the former default). Every engine chain ends with `gemini-3.8-flash` as its fallback: it reads a page only when the engines before it fail, and the engine that produced a page is stamped in its `<meta>`. The local route is `qwen3.8:27b-mlx`, a 27-billion-parameter multimodal model running on Apple MLX; it serves local-only runs and produced the KGW volumes of `kepler-prim` (catalog entry: model `qwen3.8:27b-mlx`, fallback `kimi-k2.7-code:cloud`). Its validated parameter set on this hardware (M1 Ultra Mac Studio) is:

#table(
  columns: (37.93%, 24.14%, 37.93%),
  align: (auto,auto,auto,),
  table.header([Parameter], [Value], [Rationale],),
  table.hline(),
  [`num_batch`], [512], [Reduces prefill latency from 26 s to 0.1 s],
  [`num_ctx`], [8192], [Sufficient for one PDAI page],
  [`num_predict`], [2048], [Avoids truncation on dense tables],
  [`reasoning`], [false], [Required: thinking leaks into output],
  [Workers], [1], [Optimal: 2 workers do not increase throughput on this hardware],
)
The OCR runner produces one `page_NNN.xml` per page, writes a JSONL telemetry file, and serves a live auto-refreshing dashboard at `http://localhost:<port>/ocr-monitor?volume=<vol>`. Volume succession chains run automatically (`chain_kepler.sh`: KGW 13 → 14 → … → 19), each volume executing its own pilot gate independently before batch unlock.

== Stage 5: Certify and learn
<stage-5-certify-and-learn>
The `certify-and-learn` stage is the post-batch entry in the Partitur and is #strong[not optional]. The command is:

```
pdai-audit --collection <name> --vol <vol> --all --emit-annotations
```

`--all` runs five sub-stages in sequence. Each is described in detail in § 5.

The certification criterion is: `blockers == 0 AND vis_blockers == 0`. The report then states the verdict `CERTIFIED CONDITIONAL`, otherwise `BLOCKED`, and the command exits with an error while structural blockers remain. The certificate is not yet a condition of serving: the edition viewer serves what its registry lists (§ 8.1).

== Stage 6: Corpus completion
<stage-6-corpus-completion>
After certification, the completion command audits the collection, runs the missing or stale indexing stages in dependency order and verifies each one by row counts, document coverage and a live probe query (`pdai-complete --collection <name>`, module `ai_lemma_index.collection.complete`):

+ Stanza: `ai_lemma_index.stanza.batch --changed` → Stanza page documents
+ Corpus: `ai_lemma_index.corpus.spine` → `<collection>_corpus.duckdb` (corpus/2: the sentence and token spine, the lemma table `stz_lemma` and the BM25 indexes in one file; a delta replaces only the changed documents)
+ Gazetteer: `ai_lemma_index.ner.gazetteer_store` → `gazetteer_person.parquet`

The former vector stage (`chroma_build`) is gone: the ChromaDB store was removed on 20 October 2026 and BM25 over the corpus sentences replaced it @grasshoff2026famulusdatalayer. Collections not yet switched to corpus/2 still run the old corpus build into `<collection>_stanza.duckdb` and the lemma stage into `lemma_index.parquet`.

OCR page XML and `structure.xml` belong to `ai-scholar` (its own environment); when they are missing, `pdai-complete` names the commands as gaps instead of running them. The LWgrammar lexicon and the register of individuals are built by their own commands. The final audit writes `status.json` to the collection root on PDAI-1. This file is the readiness manifest: it shows which stages are complete and whether the collection is `search_ready`, `partial` or `not_ready`.

== Stage 7: The ai-viewer endpoint
<stage-7-the-ai-viewer-endpoint>
A certified, indexed collection is served by the `ai-viewer` instance for its edition at its registered port. The viewer provides:

- #strong[Dual-pane split view]: facsimile image alongside the structured transcription, sharing one page address (`pidx`)
- #strong[Segment navigation]: the segment dropdown (letters, sections, paragraphs) is driven by `manifest.json` and allows direct navigation to any letter or section without knowing which page it occupies
- #strong[Schema-aware rendering] for every collection: the shared book sheet (`pdai-page.css` and genre sheets) and the renderer `repro-xml.js`, which preserves empty elements and marks every element with `data-el` and every attribute as `data-<attribute>`\; the collection `page.css` is the last sheet and sets measures and typography. Margins are placed in the margin at the height of their line
- #strong[The `<ai-query-chat>` web component]: a Shadow DOM component, self-contained with all CSS bundled inline, that embeds the full query interface (query, evidence, solution path) within the viewer page, firing `navigate` events to open the relevant page in the viewer
- #strong[The integrated `/audit` page]: a live quality-scoring panel for the collection's golden test set, visible in every viewer instance

= Certify and Learn
<certify-and-learn>
== The four-layer fault model
<the-four-layer-fault-model>
Every defect identified during `pdai-audit` is attributed to one of four root-cause layers. This attribution is the routing signal for the learning loop: a fault attributed to Layer 1 triggers a patch to the engine overlay of the composed prompt; a fault attributed to Layer 3 triggers a patch to the stylesheet (shared geometry in `pdai-page.css`, measures in `page.css`).

#arxivai.method-box(title: [Four-layer root-cause attribution])[
  Every fault case is attributed to one of four layers.

  *Layer 1 — Prompt / Vision LLM*: the composed prompt (page language,
  collection base, engine overlay) governing the OCR model's output. Faults here: missing `<description>` in figures, wrong element choice
  for a structural pattern, `<think>` leakage, table flattened to prose.

  *Layer 2 — Runner Normalizer*: the normalizer of the OCR runner
  (`ocr_runner.normalize` and a collection's own normalizer) that
  post-processes raw model output. Faults here: unjoined hyphen at line end,
  unclosed tag from token limit, HTML entity in text content.

  *Layer 3 — Typographic CSS*: the shared `pdai-page.css` and the collection's
  `config/page.css` governing display in `ai-viewer`. Faults here: a marginal
  in the text column, line-break fragmentation visible in rendering, wrong
  element spacing.

  *Layer 4 — DTD / Record Schema*: the grammar files. Faults here: element
  not declared, attribute value outside the closed vocabulary, `<marginal>`
  type unrecognised.
]
A fault attributed to the wrong layer cannot be resolved: a CSS fix for a prompt omission will not affect newly generated pages, and a prompt fix for a CSS gutter overflow will not affect existing transcriptions.

== The tag hierarchy protocol
<the-tag-hierarchy-protocol>
The tag hierarchy protocol is the zero-regression anchor of the pipeline. For every page in the volume, the protocol logs a content-free structural fingerprint: the S-expression tree of XML element names, their structural attributes (such as `figure[ref]`, `marginal[side,type]`, `hi[rend]`) and their nesting order, with no text content, and a SHA-256 hash of that S-expression.

```
page_042: (page (meta) (header) (main (parno) (p) (equation) (figure (caption) (description)) (p) (p)) (footnotes (footnote)))
sha256: a3f7b91c4d8e2f50...
```

When a prompt overlay is revised, `diff_tag_protocols(before, after)` compares structural hashes across all pages of the volume in under 50 ms. A fix that correctly joins hyphenated words across `<p>` boundaries shows no structural change. A fix that inadvertently drops all `<table>` elements on 34 pages shows as a mass structural delta: those 34 pages lose the `(table ...)` node from their S-expression hash.

The protocol is therefore a necessary condition for promoting an instruction change: any change that alters the structural fingerprint of a page that was previously correct must be investigated before promotion.

== Figure completion census
<figure-completion-census>
Every `<figure>` element in the volume is audited for three completion components: `@ref` (required attribute linking to the extracted image file), `<caption>` (transcribed printed caption), and `<description>` (a geometric or diagrammatic description authored by the OCR model).

Completion is expressed as four states:

- `complete_full`: both `<caption>` and `<description>` present --- standard for figures with a printed caption
- `complete_desc`: `<description>` only --- standard when the printed page has no explicit caption (common in the Astronomical Diaries)
- `caption_only`: `<caption>` present but no `<description>` --- a repair target; the model transcribed the caption but did not author a description
- `empty`: neither element --- a blocker registered as a fault case in the dossier

For a certified volume, every figure must be at `complete_full` or `complete_desc`. The figure census in the LettreAI Typst report shows the distribution across all pages and flags the page indices with `empty` figures for targeted reprocessing.

== The fault dossier
<the-fault-dossier>
The fault dossier (`learning_fault_dossier.json`) is the machine-readable artefact consumed by the learning loop. It assembles all non-trivial defects into structured `FaultCase` records. Each record carries:

- Collection, volume, page, page index (`pidx`) and printed page as coordinates, with the scan path and the viewer address of the page
- Defect codes (`SYN-*`, `SCH-*`, `VIS-*`), severity and category
- A one-sentence fault summary and a precise fault description written to enable a subsequent learning iteration to address the defect without regression, with an evidence snippet
- The root-cause layer and the model and template that produced the page
- A targeted fix directive naming the files to patch and the rule to add
- A regression anchor --- an assertion the fixed page must satisfy

A fault case from the KGW 15 dossier (abridged):

```
{
  "fault_id": "FAULT-KGW-15-PAGE_054-HYPHEN",
  "collection": "kepler-prim",
  "volume": "kgw-15",
  "pidx": 54,
  "printed_page": "55",
  "defect_codes": ["VIS-HYPHEN-SPLIT"],
  "severity": "repair",
  "category": "hyphenation",
  "layer": "Layer 1 (Prompt Overlay) & Layer 2 (Runner Normalizer)",
  "fault_summary": "Unjoined line-end hyphen before <br/> breaking word flow",
  "model": "qwen3.8:27b-mlx",
  "targeted_fix": {
    "type": "prompt_and_normalizer",
    "target_files": "collections/kepler-prim/ai-instructions/kepler-ocr.qwen38.j2, pipeline/ocr-runner/src/ocr_runner/normalize.py",
    "rule": "Prompt overlay: Enforce joining of hyphenated line-breaks in running prose. Normalizer: Add regex … in the normalizer hook."
  },
  "regression_anchor": {
    "assert_no_tag": "br[following-sibling::text()[starts-with(., '-')]]",
    "assert_continuous_prose": true
  }
}
```

== The learning bridge
<the-learning-bridge>
When `--emit-annotations` is passed to `pdai-audit`, the learning bridge (`learn_bridge.py`) posts every fault case to the ai-query server's annotation endpoint (`POST /api/annotations`, port 5113). The annotation carries the fault's coordinates, the observed defect, and the targeted fix directive.

#arxivai.method-box(title: [The learning bridge iron rule])[
  Every OCR defect that reaches the fault dossier must enter the annotation
  system and be addressed through the A/B-gated learning loop. No ad-hoc
  patch to a prompt or normalizer is permitted outside the versioned cycle.
  Code-side probes are supporting signal only; the responsible SKILL.md or
  schema file is the patch target.

  The annotation payload is always persisted locally before the HTTP post,
  so no fault data is lost when the ai-query server is unreachable. Re-ingestion
  is idempotent: the bridge keeps a ledger of the fault cases already accepted
  (`learning_annotations_sent.json`) and posts each `fault_id` once.
]
The annotation loop then:

+ Proposes a patch to the instruction file the annotation names
+ Stores the patch as a `candidate` version in `instruction_versions`
+ Runs the A/B gate: the assessment suite with the patched instruction against a baseline measured on the unpatched one
+ Promotes the version if it improves the score
+ Rejects it otherwise, restores the file and keeps the candidate for forensics

One call of the loop makes one such pass; the golden-suite loop that improves the weakest subagent is bounded at five iterations. Neither runs open-ended.

== The Typst certification report
<the-typst-certification-report>
The LettreAI Typst report (`audit_report_<vol>.pdf`) is the human-readable output of the `certify-and-learn` stage. It is generated in the LettreAI theme (Oxford blue, gold, and sandstone palette) and contains:

- #strong[Executive summary]: verdict (CERTIFIED CONDITIONAL / BLOCKED), page count, well-formed count, blocker count
- #strong[Processing architecture and DTD architecture]: the active Partitur and the structural element categories of the grammar
- #strong[Figure completion census]: four-state distribution across all figures in the volume
- #strong[Tag hierarchy protocol summary]: structural hash coverage and the deviation count from the baseline
- #strong[Structural and visual audit findings]: SYN-#emph[, CHR-], SCH-\* and VIS-\* defects tabulated by page and severity, with root-cause attribution
- #strong[Fault dossier]: all fault cases grouped by root-cause layer, with targeted fix directives and regression anchors
- #strong[CSS coherence]: selector coverage analysis showing elements and attribute values with no CSS rule
- #strong[Certification and recommended action plan]: ordered by layer, with file paths and rule text

= The Contextual Data Stores
<the-contextual-data-stores>
A certified PDAI page is the ground layer. The contextual data stores are what make it queryable. After `pdai-complete --collection <name>`, a collection maintains the following stores:

== Corpus file and famulus file
<corpus-file-and-famulus-file>
A collection keeps two built DuckDB files at its root (`/Volumes/PDAI-1/pdai-collections/<name>/`); which file a role resolves to is decided by `ai_lemma_index.stores` from a recorded state per collection @grasshoff2026famulusdatalayer:

- `<name>_corpus.duckdb` (corpus/2), the sentence corpus, rebuilt from the Stanza page documents and the page XML: `stz_docs`, `stz_pages`, `stz_page_text` (the printed text of each page), `stz_sentences` (sentences of the Stanza parse, keyed by the triple `(doc_id, pidx, sent_idx)` --- the sentence key --- with `sent_key` and the lemmatised sentence `lemma_text`), `stz_tokens` (dependency head with its lemma and part of speech, the feature string, character offsets and the NER tag as columns), `stz_entities` (entity spans with type and surface form), the views `stz_deps`, `stz_feats` and `lemma_index`, the lemma table `stz_lemma`, and the BM25 indexes over the sentences
- `<name>_famulus.duckdb`, everything derived for answering, proving and displaying: for collections with a letter register `letters_resolved`, one row per letter with sender and recipient as register identifiers, date, place, regest and source pages, and the `ent_*` projection of the register (`ent_canonical`, the canonical entity forms with their spellings); the sidecar of the Famulus Query Language for the correspondence collections (letters, letter pages, footnotes, mentions and topics as `fq_*` tables); the evidence space (`fx_*`), the sentence records (`sr_*`, `fnf_*`), the dossier projection (`do_*`) and the famulus reference (`fr_*`)

These two files replace `<name>_stanza.duckdb`, the per-document `index.duckdb` files, `lemma_index.parquet`, `<name>_fql.duckdb`, `<name>_fx.duckdb`, `<name>_satz.duckdb`, `<name>_dossier.duckdb`, the main `<name>.duckdb` and the ChromaDB store; on 20 October 2026, 17 collections were switched and eleven still kept the old layout.

#arxivai.method-box(title: [The sentence key invariant])[
  A PDAI sentence is keyed by the triple (doc_id, pidx, sent_idx). The
  sent_idx restarts on every page, so a key without pidx is not a sentence
  key — it undercounts sentences and invents cross-page co-occurrences. Every
  query that refers to sentence position must include pidx.
]
The distinction between correspondence partners (senders and receivers of letters) and persons merely mentioned in text bodies is maintained by the schema: `letters_resolved.sender_id`/`.recipient_id` for the epistolary channel, `stz_entities.text` for body mentions. A query that conflates the two channels produces wrong results.

== Lemma postings and BM25
<lemma-postings-and-bm25>
The lemma table `stz_lemma` of the corpus file (formerly the separate file `lemma_index.parquet`, whose six columns it keeps) is the fast filter of the query pipeline. It maps each lemma to the set of postings `(doc_id, pidx, tid)` --- document, page index, and token position --- where it appears. The table is physically ordered by lemma, so postings for a query's conjunctive-normal-form lemma expression are retrieved in milliseconds by DuckDB's zone-map pruning (p95 3.0 ms on karman-nachlass against 5.3 ms for the parquet file @grasshoff2026famulusdatalayer).

Full-text recall is a BM25 index over the sentence text and `lemma_text`\; for German and French a second index over character 4-grams of the words is fused with it by reciprocal rank fusion, because compound words defeat word-level BM25. It replaced the vector store, at a measured loss: on 200 paraphrase probes it finds the source sentence in the top 50 for 0.830 of them (Latin 0.94, German 0.74, English 0.84, French 0.80) and in the top 10 for 0.725, where the bge-m3 vector store did for 0.895 and 0.890.

Two counting modes are distinguished:

- `canonical`: the lemma only (as assigned by Stanza's Latin or German model)
- `family`: canonical plus OCR variants registered in the collection's alias table

These modes give systematically different counts and must be declared with every cited frequency. The token-position column in `stz_lemma` (as in the former `lemma_index.parquet`) is `tid`, not `token_idx`.

== Gazetteer
<gazetteer>
The gazetteer stores person names, place names, and domain-specific concepts with their collection-specific variant spellings, as the Parquet files `gazetteer_person.parquet`, `gazetteer_place.parquet` and `gazetteer_domain.parquet`. They are built by `ai_lemma_index.ner.gazetteer_store`\; since GLiNER2.5 left the NER pipeline on 8 October 2026, `pdai-complete` rebuilds only the person gazetteer, and the place and domain gazetteers keep their last build. The gazetteer serves the `entity-fast-filter` subagent, which filters candidate documents by person or place name across all `entity_variants` spellings before any full LLM pass.

Person names must never appear in `concept_terms` or terminology signals. Persons are on the entity-registry channel only; concept terms are domain vocabulary (mathematical, astronomical, philosophical). This channel separation is enforced by the query pipeline's entity fast-filter, which returns zero results if a person name is submitted to the concept-term lookup.

== LWgrammar lexicon
<lwgrammar-lexicon>
The LWgrammar (`<collection>_lwgrammar.duckdb`) stores OED-form usage entries for lemmas that appear frequently enough to merit a dictionary treatment (threshold: \~10 occurrences). Each entry has a headword, forms, numbered senses, and dated quotations linked to their `(doc_id, pidx, sent_idx)` sentence keys.

Entries are compiled and revised by `ai_lemma_index.lwgrammar`, which calls `gemini-3.8-flash` following the OED instructions and the reference `time` entry, and records the SHA-256 of each prompt in the revision log. The lexicon is the resource for the dictionary operation in AI Erasmus: a scholar who asks how Kepler uses #emph[lex] is asking for the LWgrammar entry for #emph[lex] in the scope `kepler-prim`.

== Register of individuals and partitur register
<register-of-individuals-and-partitur-register>
The global register of individuals (`pdai-collections/_register/register.duckdb`) gives one identifier to each person, place and institution across all collections; on 20 October 2026 it held 15,452 persons, 4,063 places and 5,094 institutions as active entries. Collection builders deliver only the surface spellings they find; whether two spellings name the same individual is decided by an agent following the identity instruction (`pdai/skills/register/identitaet/SKILL.md`) and checked by a second reader, and owner corrections outrank both. The older entity registry of the query layer (`entity_registry.duckdb` in ai-query) still serves the `entity-fast-filter` and `correspondence-roster` specialists.

The Partitur register (`partitur_register.duckdb`, shared between the ai-query server and the audit CLI) stores goal contracts, plan paths, stage definitions, and result specifications for every registered Partitur. It is opened with short-lived, per-operation connections --- a permanent write connection causes silent lock starvation that manifests as annotation and learning-loop failures.

== The reference layer
<the-reference-layer>
The stores above serve queries; they do not yet say how an item is to be addressed and cited. The reference layer (package `pdai/famulus-reference`) turns a collection into a #emph[famulus reference]: every document, page and segment receives a stable address, a citation, a provenance chain and validation records. The address grammar is

```
fr:<collection>/<doc>[/<pidx:04d>[/<segment>]]
```

for instance `fr:hartmann/bsb00028171/0137/print1`\; a sentence is addressed by `s` and three digits in reading order below its page. Global identifiers (URN, DOI, GND, shelfmark, IIIF) are kept beside the address, not in its place.

`famulus-reference build` reads a collection folder without changing it and writes the schema `famulus-reference/1` (the `fr_*` tables) into `<collection>_famulus.duckdb`, beside the other derived tables: collection, source, document, page, segment, text layers, entities and mentions, citation, provenance, validation, identifiers, DTDs and harvest records. A citation is derived only from `biblio.xml` or the collection's descriptor; a missing field is recorded as a citation gap and never filled by inference. `famulus-reference export` then writes the collection's Parquet files into one DuckLake (catalog `pdai-collections/_lake/famulus.ducklake`, data under `<collection>/_lake/`). Only mac-studio writes; every other reader attaches a read replica that each write session publishes. On 20 October 2026 eight collections were in the lake (`enestroem`, `eulerOO`, `eulerOO-IVA`, `hartmann`, `hevelius`, `karman-context`, `karman-nachlass`, `pirckheimer`).

One MCP server (port 5135) answers for all of them: it describes a collection, returns items, DTDs and provenance, cites an item as BibTeX, CSL-JSON or text with the nearest citable ancestor and a locator, grounds a phrase lexically to `fr:` addresses, and runs guarded read-only SQL. Every citation names the lake snapshot it was drawn from, and cited snapshots never expire, so a citation can be resolved later against the state it was made from; citing and grounding calls record a run whose `run_id` reproduces the items behind a past answer. For catalogs the same server speaks OAI-PMH 2.0, schema.org/DCAT JSON-LD and IIIF Presentation 3. Three limits are part of the present state: `fr_ground` ranks by token frequency, not by a full-text index; the validation records do not yet carry the `pdai-audit` certificate; and the per-page model and partitur of `<meta>` are not yet parsed into `fr_provenance`.

= Case Studies at Scale
<case-studies-at-scale>
== KGW 15: a certified volume
<kgw-15-a-certified-volume>
Kepler's Gesammelte Werke volume 15, the letters of 1604--1607, has 567 pages in the collection. Its letters carry astronomical computation: 38 of its pages hold `<table>` elements, and the volume includes 89 figures.

After full batch OCR and the `certify-and-learn` stage (re-audit of 26 September 2026, `pdai/collections/kepler-prim/quality/kgw-15/`):

- Structural blockers: 0 (29 pages with repairs)
- Visual blockers: 0
- Verdict: `CERTIFIED CONDITIONAL`
- Tag hierarchy protocol: 567 fingerprints
- Figure completion: 89/89 (19 `complete_full`, 70 `complete_desc`)
- Fault cases registered: 9 (three tables, three mathematical expressions, one hyphenation, one epistolary and one marginal case; none blocking certification)

The first audit, of 22 August, had posted 25 annotations to the ai-query learning loop (`learning_annotations_payload.json`); the re-audit of 26 September registered the nine cases above.

== Pirckheimer (354 letters): correspondence goldens
<pirckheimer-354-letters-correspondence-goldens>
The Pirckheimer correspondence collection holds 354 letters and 78 persons in `pirckheimer.duckdb` (`letters_resolved` + `ent_canonical` tables). Its golden queries (`pdai/agents/ai-query/partitur/goldens/manifest.yaml`) fix what a correct answer must contain:

#emph[How many correspondents did Pirckheimer have between 1520 and 1530?] → oracle: between 70 and 85 distinct correspondents.

#emph[What is the subject of the letters from Schöner between 1520 and 1527?] → oracle: 11 letters, subject the #emph[Saphea].

The retrieval trails of these two queries --- classification, collection selection, entity filter and validation over DuckDB, without a language model --- took 248.2 ms and 309.9 ms in the golden run of 15 August 2026 (`golden_runs/reports/report.md`). The goldens test that the register maps correspondent name variants to one individual, that `letters_resolved` covers the full date range, and that senders and recipients are kept apart from persons merely mentioned in a letter.

== Astronomical Diaries (ADRT 1--7): transliteration at scale
<astronomical-diaries-adrt-17-transliteration-at-scale>
The Diaries collection spans 7 volumes with 3,853 pages. Of these, 2,617 were read by `qwen3.8:27b-mlx` under the instruction family `diaries-translit-ocr` (a base layer with engine layers for `qwen38`, `kimi` and `gemini38`), and 1,145 carry the text layer of the source PDF, as their `<meta model>` states.

The unattended volume chain (`chain_diaries_all.sh`) processes ADRT 1 through 7 in succession, each volume executing its pilot gate independently. The live monitor at `http://localhost:5112/ocr-monitor?volume=adrt<N>` reports worker-aware throughput metrics from the runner's `InflightTracker`.

A critical invariant: when page indices are shifted (e.g., blank flyleaves are removed from ADRT 1), every downstream reference --- page folders, `pidx` values in XML, `spread_map.json`, DuckDB tables, and golden queries --- must be updated. Shift arithmetic alone is insufficient; an independent correlation control (`verify_pidx_correlation.py`) confirms that claimed OCR content matches expected text at the new `pidx` before the migration is accepted.

= The Edition Registry and the Viewer Contract
<the-edition-registry-and-the-viewer-contract>
== The delivery contract
<the-delivery-contract>
A volume is to be delivered to the ai-viewer endpoint when three conditions are simultaneously satisfied:

+ `pdai-audit` certification: `blockers == 0 AND vis_blockers == 0`
+ `pdai-complete` readiness: all corpus stages complete (`status.json`)
+ Edition registry entry: the volume appears as a document, with alias, path and collection, in the edition's registry `/Volumes/PDAI-1/editions/<edition>/registry.json`

Of the three, only the third is enforced by the software today: the viewer serves what the registry lists. The supervisor `editions_supervisor.py` (launchd job `com.ai-platform.editions`, under KeepAlive) is a process supervisor. It keeps 24 edition viewers and three services (the glider search, the editions landing and the Kármán viewer) running, restarts a child with backoff and probes each over HTTP; it does not check certification or readiness. The landing manifest `editions.yaml` of ai-viewer maps each edition to its collections, port and registry. Making the certificate and the readiness manifest conditions of registration is open work.

== The edition viewer
<the-edition-viewer>
The `ai-viewer` instance for a collection is a FastAPI server that serves:

- `GET /doc/<alias>/viewer`: the split-view page with facsimile and transcription
- `GET /api/collection/<collection>/page.css`: the collection's overlay stylesheet
- `GET /doc/<alias>/segments/<type>`: the manifest-driven segment navigation
- `GET /audit`: the integrated quality-scoring panel for the collection's golden set

The DOM-based renderer (`repro-xml.js`) renders every page. It produces a tree of elements with `data-el="tagname"` and `data-<attribute>="value"` that CSS can fully target; a marginal is an inline span that the shared sheet moves into the margin while it keeps the height of its line. The `@style` collision that plagued the legacy string transform is absent: every XML attribute is a `data-` attribute on the DOM node.

The `<ai-query-chat>` web component runs in Shadow DOM with all CSS bundled inline (`?inline` import into the shadow root `<style>` tag). This makes the query interface self-contained across all viewer instances --- no shared stylesheet required.

== Port registry
<port-registry>
Each edition viewer runs at a registered port, mapped by the platform registry:

#table(
  columns: 2,
  align: (auto,auto,),
  table.header([Collection], [Viewer port],),
  table.hline(),
  [Enestroem], [5100],
  [Euler OO], [5103],
  [Kepler], [5104],
  [Copernicus], [5105],
  [Waefler], [5106],
  [Euler OO IVA], [5107],
  [Pirckheimer], [5108],
  [Kirch], [5109],
  [Editions landing], [5111],
  [Diaries], [5112],
  [Kepleriana Munich 2024], [5127],
  [Medieval-Sec (Bischoff)], [5128],
  [Brahe OO], [5136],
  [Hevelius], [5137],
  [Kármán context], [5140],
  [Kármán archive], [5725],
)
The ai-query lab (port 5113) is the query-resolution dev server, not a viewer instance. The famulus-reference MCP server (port 5135) serves the reference layer; it is not a viewer instance either.

= Implications for Computational Epistemology
<implications-for-computational-epistemology>
== The page as the unit of evidence
<the-page-as-the-unit-of-evidence>
The unit of evidence in the PDAI system is the page: one raster image plus its structured transcription, sharing one address (`pidx`); in the reference layer that address is the page's `fr:` address. Every query resolves to pages; every corpus operation --- lemma lookup, entity filter, dictionary construction --- takes place over page sentences.

This is an architectural commitment, not a convenience. A claim about what Kepler wrote can be checked against its source in one click: the page address opens the facsimile alongside the transcription, providing the same ground a printed apparatus would offer, reached in milliseconds. A system whose answers ended in paraphrase --- summarising what Kepler wrote without a path back to the page --- would leave scholarly dispute nowhere to stand.

The audit system states this commitment as a gate. A page that fails certification is not a page; it is an unresolved production artefact. The `blockers == 0` criterion is not a quality aspiration but a binary gate: either the page's structural transcription is trustworthy, or it does not belong in the corpus.

== Grounding Evidence vs Sekundärevidenz
<grounding-evidence-vs-sekundärevidenz>
The PDAI system distinguishes two kinds of scholarly evidence. #emph[Grounding Evidence] is the direct textual or observational basis of a historical claim: a letter page, an observation record, a theorem statement. #emph[Sekundärevidenz] is the interpretive layer: a monograph's account of what the letter says, a commentary's gloss of the theorem.

PDAI pages are Grounding Evidence by design: they are transcriptions of the original sources, validated against the original typography, with provenance recording the model and instruction that produced them. They do not interpret; they transcribe. The query layer reads Grounding Evidence to answer scholarly questions; it does not substitute interpretation for the source.

This distinction matters for computational epistemology because the two kinds of evidence have different epistemic status. A claim grounded in a letter page can be reproduced and disputed at the level of the facsimile; a claim grounded in a secondary account inherits all the interpretive decisions of that account. The PDAI infrastructure makes Grounding Evidence computationally accessible without collapsing it into secondary interpretation.

The reference layer carries the distinction into justification. The library famulus-proof takes an `fr:` address verbatim as the address of a ground, and its resolver returns the item, its text layers, citation and provenance from the lake; an address that does not resolve is reported as a gap, never read as an empty pass. A ground that turns out to be wrong --- a misread page, a wrong register entry --- makes the argument resting on it unsound and withdraws the justification of its conclusion; it does not make the conclusion false. The address tells which conclusions lose their justification when a page is corrected.

== Reproducibility and provenance
<reproducibility-and-provenance>
Every PDAI page records which model, instruction template, and date produced its transcription. This makes PDAI corpora reproducible in a strong sense: not merely that the data is available, but that the production process can be re-run with identical configuration and checked to produce the same structural output.

The instruction versioning system in the learning loop ensures that a change to an instruction template is recorded with a version number, a change rationale, and the results of the A/B test that promoted it. A scholar who cites results computed over a PDAI corpus can cite the instruction version and the model alongside the document set; a future researcher can check whether a reprocessed corpus with an updated instruction produces the same structural fingerprints on the regression anchor pages. A citation drawn from the reference layer names the lake snapshot it was made from, and that snapshot is kept, so the cited state of the corpus remains available after it has changed.

This provenance chain is what distinguishes PDAI-certified pages from uncertified OCR output. Uncertified output may be accurate; it does not carry the structural audit, the fault dossier, or the regression guarantee that make it epistemically reliable as Grounding Evidence.

== Computational engagement with primary sources
<computational-engagement-with-primary-sources>
Traditional scholarship involves locating documents, transcribing relevant passages, translating foreign-language content, manually extracting structured data, and building indices and concordances by hand. With a PDAI-certified corpus, these steps become computational: the corpus is queried by lemma, entity, date, or document type; translation layers accumulate beside the page (`page_NNN_translation_<language>.xml`) without overwriting transcriptions; entity extraction is a stanza pipeline stage, not a manual operation; indices are built by `bib_build` and the LWgrammar builder, not by hand.

This does not replace close reading. It augments it. The scholar can still read Kepler's Latin --- and the viewer shows the facsimile alongside the transcription --- but they can also query the 934 letters of the Kepler register by correspondent and topic, compare the use of #emph[lex] across the Kepler correspondence and the Euler treatises, and trace the concept of #emph[harmonia] across 26 volumes of Gesammelte Werke.

The infrastructure handles the mechanical labour --- transcription, validation, indexing, registry maintenance --- so that scholarly judgment can be spent where only judgment serves.

= Conclusion
<conclusion>
PDAI is an infrastructure for treating historical scholarly documents as Grounding Evidence in computational inquiry. Its format encodes the nine structural categories of historical scholarly typography in a grammar that is validated per page. Its production pipeline governs the path from PDF to certified corpus through a Partitur that includes a mandatory audit and learning stage. Its contextual data stores --- DuckDB corpus, lemma postings, gazetteer, LWgrammar lexicon, register of individuals --- make certified pages answerable to structured scholarly queries. Its reference layer gives every item an `fr:` address, a citation from recorded bibliographic data and a provenance record, so that an answer can name the page and the snapshot it rests on.

The certify-and-learn system closes the production loop: every defect is attributed to a root-cause layer, registered as a versioned learning case, and resolved through an A/B-gated instruction improvement. The tag hierarchy protocol provides the regression guarantee. The fault dossier provides the precise descriptions that make improvement possible without inadvertent structural degradation.

On 20 October 2026 the PDAI volume held about 179,000 page XML files in 51 collections, 14,864 of them in the Kepler collection alone. Of the 50 collections with a readiness manifest, 25 are `search_ready`, 5 `partial` and 19 `not_ready`\; the manifest of the Diaries collection records only its OCR state. Eight collections are in the DuckLake of the reference layer. The remainder are at various stages of production.

The infrastructure is open and documented. The format specification (`pdai/docs/`), the production pipeline skills (`pdai/skills/pipeline/`, including the intake skill `pdai-collection-intake`) and the reference package (`pdai/famulus-reference/README.md`) are the primary references. The Quarto documentation book at `pdai/docs/_site/` is generated from the workspace and serves as the canonical reference for anyone working with the PDAI system.

= Acknowledgments
<acknowledgments>
I thank the libraries and archives whose digitisation programs made the source materials available. The Quarto project enabled the documentation system; DuckDB and DuckLake provided the query foundation; the Ollama project made local inference practical. This work was supported by the Humboldt-Universität zu Berlin (Philosophisches Institut) and the Forschungsinstitut des Deutschen Museums, München.

#bibliography(("ai-scholar-bibliography.bib"))