// Simple numbering for non-book documents
#let equation-numbering = "(1)"
#let callout-numbering = "1"
#let subfloat-numbering(n-super, subfloat-idx) = {
  numbering("1a", n-super, subfloat-idx)
}

// Theorem configuration for theorion
// Simple numbering for non-book documents (no heading inheritance)
#let theorem-inherited-levels = 0

// Theorem numbering format (can be overridden by extensions for appendix support)
// This function returns the numbering pattern to use
#let theorem-numbering(loc) = "1.1"

// Default theorem render function
#let theorem-render(prefix: none, title: "", full-title: auto, body) = {
  if full-title != "" and full-title != auto and full-title != none {
    strong[#full-title.]
    h(0.5em)
  }
  body
}
// Some definitions presupposed by pandoc's typst output.
#let content-to-string(content) = {
  if content.has("text") {
    content.text
  } else if content.has("children") {
    content.children.map(content-to-string).join("")
  } else if content.has("body") {
    content-to-string(content.body)
  } else if content == [ ] {
    " "
  }
}

#let horizontalrule = line(start: (25%,0%), end: (75%,0%))

#let endnote(num, contents) = [
  #stack(dir: ltr, spacing: 3pt, super[#num], contents)
]

#show terms.item: it => block(breakable: false)[
  #text(weight: "bold")[#it.term]
  #block(inset: (left: 1.5em, top: -0.4em))[#it.description]
]

// Some quarto-specific definitions.

#show raw.where(block: true): set block(
    fill: luma(230),
    width: 100%,
    inset: 8pt,
    radius: 2pt
  )

#let block_with_new_content(old_block, new_content) = {
  let fields = old_block.fields()
  let _ = fields.remove("body")
  if fields.at("below", default: none) != none {
    // TODO: this is a hack because below is a "synthesized element"
    // according to the experts in the typst discord...
    fields.below = fields.below.abs
  }
  block.with(..fields)(new_content)
}

#let empty(v) = {
  if type(v) == str {
    // two dollar signs here because we're technically inside
    // a Pandoc template :grimace:
    v.matches(regex("^\\s*$")).at(0, default: none) != none
  } else if type(v) == content {
    if v.at("text", default: none) != none {
      return empty(v.text)
    }
    for child in v.at("children", default: ()) {
      if not empty(child) {
        return false
      }
    }
    return true
  }

}

// Subfloats
// This is a technique that we adapted from https://github.com/tingerrr/subpar/
#let quartosubfloatcounter = counter("quartosubfloatcounter")

#let quarto_super(
  kind: str,
  caption: none,
  label: none,
  supplement: str,
  position: none,
  subcapnumbering: "(a)",
  body,
) = {
  context {
    let figcounter = counter(figure.where(kind: kind))
    let n-super = figcounter.get().first() + 1
    set figure.caption(position: position)
    [#figure(
      kind: kind,
      supplement: supplement,
      caption: caption,
      {
        show figure.where(kind: kind): set figure(numbering: _ => {
          let subfloat-idx = quartosubfloatcounter.get().first() + 1
          subfloat-numbering(n-super, subfloat-idx)
        })
        show figure.where(kind: kind): set figure.caption(position: position)

        show figure: it => {
          let num = numbering(subcapnumbering, n-super, quartosubfloatcounter.get().first() + 1)
          show figure.caption: it => block({
            num.slice(2) // I don't understand why the numbering contains output that it really shouldn't, but this fixes it shrug?
            [ ]
            it.body
          })

          quartosubfloatcounter.step()
          it
          counter(figure.where(kind: it.kind)).update(n => n - 1)
        }

        quartosubfloatcounter.update(0)
        body
      }
    )#label]
  }
}

// callout rendering
// this is a figure show rule because callouts are crossreferenceable
#show figure: it => {
  if type(it.kind) != str {
    return it
  }
  let kind_match = it.kind.matches(regex("^quarto-callout-(.*)")).at(0, default: none)
  if kind_match == none {
    return it
  }
  let kind = kind_match.captures.at(0, default: "other")
  kind = upper(kind.first()) + kind.slice(1)
  // now we pull apart the callout and reassemble it with the crossref name and counter

  // when we cleanup pandoc's emitted code to avoid spaces this will have to change
  let old_callout = it.body.children.at(1).body.children.at(1)
  let old_title_block = old_callout.body.children.at(0)
  let children = old_title_block.body.body.children
  let old_title = if children.len() == 1 {
    children.at(0)  // no icon: title at index 0
  } else {
    children.at(1)  // with icon: title at index 1
  }

  // TODO use custom separator if available
  // Use the figure's counter display which handles chapter-based numbering
  // (when numbering is a function that includes the heading counter)
  let callout_num = it.counter.display(it.numbering)
  let new_title = if empty(old_title) {
    [#kind #callout_num]
  } else {
    [#kind #callout_num: #old_title]
  }

  let new_title_block = block_with_new_content(
    old_title_block,
    block_with_new_content(
      old_title_block.body,
      if children.len() == 1 {
        new_title  // no icon: just the title
      } else {
        children.at(0) + new_title  // with icon: preserve icon block + new title
      }))

  align(left, block_with_new_content(old_callout,
    block(below: 0pt, new_title_block) +
    old_callout.body.children.at(1)))
}

// 2023-10-09: #fa-icon("fa-info") is not working, so we'll eval "#fa-info()" instead
#let callout(body: [], title: "Callout", background_color: rgb("#dddddd"), icon: none, icon_color: black, body_background_color: white) = {
  block(
    breakable: false, 
    fill: background_color, 
    stroke: (paint: icon_color, thickness: 0.5pt, cap: "round"), 
    width: 100%, 
    radius: 2pt,
    block(
      inset: 1pt,
      width: 100%, 
      below: 0pt, 
      block(
        fill: background_color,
        width: 100%,
        inset: 8pt)[#if icon != none [#text(icon_color, weight: 900)[#icon] ]#title]) +
      if(body != []){
        block(
          inset: 1pt, 
          width: 100%, 
          block(fill: body_background_color, width: 100%, inset: 8pt, body))
      }
    )
}



// arxivai — Quarto Typst template partial.
// Replaces Quarto's default `article()` with the arxivai preprint layout.
// Both packages are importable from raw ```{=typst}``` blocks in the body:
//   #arxivai.operation-box(...)   #arxivai.app-figure(...)   #lettreai.badge(...)

#import "lettreai.typ" as lettreai
#import "arxivai.typ" as arxivai

// ---------------------------------------------------------------------------
// Re-theme Quarto's automatic callouts (same contract as lettreai: block
// nesting kept structurally identical to Quarto's default so the crossref
// figure show rule in definitions.typ keeps working).
// ---------------------------------------------------------------------------
#let callout(
  body: [],
  title: "Callout",
  background_color: rgb("#dddddd"),
  icon: none,
  icon_color: black,
  body_background_color: white,
) = {
  let accent = if icon_color == black { lettreai.colors.oxford-blue } else { icon_color }
  block(
    breakable: false,
    fill: lettreai.tint(accent, 0.94),
    stroke: (left: 3pt + accent, rest: 0.5pt + lettreai.tint(accent, 0.5)),
    width: 100%,
    radius: 3pt,
    block(
      inset: 1pt,
      width: 100%,
      below: 0pt,
      block(
        fill: lettreai.tint(accent, 0.84),
        width: 100%,
        inset: 8pt,
      )[#if icon != none [#text(accent, weight: 900)[#icon] ]#text(fill: lettreai.shade(accent, 0.25), weight: 700)[#title]]) +
      if (body != []) {
        block(
          inset: 1pt,
          width: 100%,
          block(fill: body_background_color, width: 100%, inset: 8pt, body))
      }
    )
}

// ---------------------------------------------------------------------------
// arXiv preprint article — same signature contract as Quarto's default,
// extended with keywords and running-title.
// ---------------------------------------------------------------------------
#let article(
  title: none,
  subtitle: none,
  authors: none,
  keywords: (),
  running-title: none,
  book: false,
  date: none,
  abstract-title: none,
  abstract: none,
  thanks: none,
  cols: 1,
  lang: "en",
  region: "US",
  font: none,
  fontsize: 10.5pt,
  title-size: 1.7em,
  subtitle-size: 1.15em,
  heading-family: none,
  heading-weight: "bold",
  heading-style: "normal",
  heading-color: black,
  heading-line-height: 0.65em,
  mathfont: none,
  codefont: none,
  linestretch: 1,
  sectionnumbering: none,
  linkcolor: none,
  citecolor: none,
  filecolor: none,
  toc: false,
  toc_title: none,
  toc_depth: none,
  toc_indent: 1.5em,
  doc,
) = {
  // PDF metadata.
  set document(title: title)
  set document(
    author: authors.map(a => lettreai.to-string(a.name)).join(", ", last: " & "),
  ) if authors != none and authors != ()
  set document(
    keywords: keywords.map(k => lettreai.to-string(k)),
  ) if keywords != none and keywords != ()

  set par(justify: true, leading: linestretch * 0.65em)
  set text(lang: lang, region: region, size: fontsize)
  show math.equation: set text(font: mathfont) if mathfont != none

  set heading(numbering: sectionnumbering)

  // Running header + footer; book mode gets chapter-aware heads and the
  // O'Reilly outer-margin geometry.
  let rt = if running-title != none { running-title } else if title != none { title } else { [] }
  set page(
    header: if book { arxivai.book-header() } else { arxivai.arxiv-header(rt) },
    footer: if book { arxivai.book-footer() } else { arxivai.arxiv-footer() },
  )
  set page(margin: arxivai.book-margin) if book

  // Resolve fonts: explicit Quarto fonts win, else theme defaults.
  let resolved-font = if font != none { font } else { lettreai.body-font }
  let resolved-code = if codefont != none { codefont } else { lettreai.mono-font }
  let resolved-heading-color = if heading-color != black { heading-color } else { lettreai.colors.oxford-blue }
  let resolved-link = if linkcolor != none { rgb(lettreai.to-string(linkcolor)) } else { lettreai.colors.oxford-blue }

  // Paint everything with the shared theme, then the figure rules.
  show: lettreai.theme-rules.with(
    fonts: resolved-font,
    code-font: resolved-code,
    heading-color: resolved-heading-color,
    link-color: resolved-link,
  )
  show: arxivai.arxiv-figure-rules
  // Book chapter rule is the innermost show, so it transforms level-1
  // headings FIRST (inner rules apply first), replacing them with chapter
  // openers before the theme's h1 rule could wrap them in a container.
  // Synthetic non-outlined headings (e.g. the outline's own title) are
  // excluded inside book-heading-rules.
  show: doc2 => if book { arxivai.book-heading-rules(doc2) } else { doc2 }

  show ref: set text(fill: rgb(lettreai.to-string(citecolor))) if citecolor != none

  // Preprint front matter.
  arxivai.arxiv-title-block(
    title: title,
    subtitle: subtitle,
    authors: authors,
    date: date,
  )
  if abstract != none {
    arxivai.arxiv-abstract(
      abstract,
      abstract-title: if abstract-title == none { [Abstract] } else { abstract-title },
    )
  }
  arxivai.arxiv-keywords(keywords)

  // Book mode: the front matter is a title page of its own.
  if book { pagebreak(weak: true) }

  // Optional table of contents (its own page in book mode).
  if toc {
    block(above: 0.5em, below: 2em, lettreai.lettreai-outline(
      title: if toc_title == none { auto } else { toc_title },
      depth: toc_depth,
      indent: toc_indent,
    ))
    if book { pagebreak(weak: true) }
  }

  if cols > 1 { columns(cols, doc) } else { doc }
}
#let brand-color = (:)
#let brand-color-background = (:)
#let brand-logo = (:)

#set page(
  paper: "a4",
  margin: (x: 2.4cm,y: 2.6cm,),
  numbering: "1",
  columns: 1,
)

#show: doc => article(
  title: [The Famulus data layer],
  subtitle: [One corpus file and one derived file per collection, measured],
  authors: (
    ( name: [Gerd Graßhoff],
      affiliation: [Philosophisches Institut, Humboldt-Universität zu Berlin, Forschungsinstitut des Deutschen Museums],
      email: [gerd.grasshoff\@hu-berlin.de],
      orcid: [0000-0002-4450-357X] ),
    ),
  abstract: [The PDAI collections of Famulus were served from 2,655 DuckDB files (63.5 GB), a ChromaDB vector store per collection (17.6 GB) and per-document intermediates. Most of it repeated the same data: every token was stored three to four times, the paragraphs of a correspondence three times, the dossier facts three times, and 29.6 GB of per-document databases fed a corpus that one parse of the page documents yields directly. This paper describes the layer that replaced it: per collection one corpus file, rebuilt from the Stanza page documents and carrying a BM25 index, and one file for everything derived from it; two small stores that services write at query time stay apart. On 28 collections the new corpus file reproduces every table of the old one, row for row, or the difference was traced to a stale old file (among them 454 pages of the Euler--Goldbach correspondence that the old incremental build never added). The large corpus files are 5--9 times smaller and build in seconds to minutes; of the 76 recorded SQL statements of the FQL bench 72 return the same rows, and the other four differ for stated reasons. The vector store was removed. On 200 paraphrase probes BM25 with a character 4-gram index for German and French finds the source sentence in the top 50 for 0.830 of them, the vector store did for 0.895, and in the top 10 for 0.725 against 0.890; that loss is stated, not hidden.

],
  abstract-title: "Abstract",
  font: ("Source Sans 3",),
  sectionnumbering: "1.1",
  toc_title: [Table of contents],
  toc_depth: 3,
  doc,
)

= The starting point, measured
<the-starting-point-measured>
Famulus answers questions to scholarly collections from evidence it can address: a page, a sentence, a letter, a figure @grasshoff2026famulusreasoning@grasshoff2026aischolar. Each collection on the PDAI volume had grown a database per purpose. A measured inventory of the volume on 20 October 2026 (#ref(<tbl-inventar>, supplement: [Table]), left columns; the script `dblayer_inventar.py` walks the volume, nothing is estimated) counted 2,655 DuckDB files @raasveldt2019duckdb with 63.5 GB, a ChromaDB store per searchable collection with 17.6 GB, 145,656 Stanza page documents @qi2020stanza with 19.0 GB and as many CoNLL-U exports with 3.2 GB.

A read-only survey of the code that writes and reads these files found three kinds of overhead. #emph[Duplication]: a token was a row of the corpus table `stz_tokens`, again a row of `stz_deps` (one row per token), again in `lemma_index.parquet`, again in the document's own `index.duckdb`, and its morphological features once more as one row per feature in `stz_feats`\; in karman-nachlass the dossier facts stood in the dossier projection, in the famulus-reference file and in the lake. #emph[Dead structures]: tables written by a builder and read by no code (`fq_segments`, `fq_tag_topic`, `fx_formen`, `sr_concept_up`, `do_nennung`, `do_querverweis`, the postings of an early aggregation step). #emph[Hazards]: a step that wrote a document's page text deleted the whole per-document file and with it the parsed tables; the evidence-space tables were written into the published FQL file, which the next FQL publication replaced; and the multiprocessing pool of the Stanza batch waited forever when a worker process died.

= Principles
<principles>
+ #strong[One spine, derived elsewhere.] The linguistic spine of a collection --- documents, pages, page text, sentences, tokens with their dependency and features, named entities, the lemma index and the full-text index --- is one file, `<c>_corpus.duckdb`. Everything built from it or from the page XML for answering, proving and displaying is one other file, `<c>_famulus.duckdb`.
+ #strong[Rebuildable from files.] The corpus file is built from the Stanza page documents and the page XML alone; no database is its input. A full build replaces it; a delta replaces the changed documents.
+ #strong[Publication by swap.] Every writer works on a copy (an APFS clone, which costs nothing until blocks change) and replaces the published file atomically after its own gates passed. A reader never sees a half-written file, and a failed build leaves the published file untouched.
+ #strong[One lock per collection.] Corpus runs take an exclusive lock on the collection, writers of single pages (OCR) a shared one; two Stanza runs on one collection can no longer interleave.
+ #strong[Explicit gaps.] A missing index or store is an error with a name, never an empty result.
+ #strong[No model text in the evidence.] The page text of the corpus is the printed text of `<main>`\; descriptions written by a model (`description`, `figDesc`, `meta`) do not enter it.
+ #strong[Short-lived connections.] Every reader opens read-only for one operation.

Two stores do not fit principle 3 and stay files of their own: the LWgrammar usage store, whose search counts the search service increments while answering, and the concept cache of ai-query, which it fills at query time. A file that is only ever replaced whole cannot take such writes.

= The corpus file
<the-corpus-file>
`<c>_corpus.duckdb` (corpus/2) has one token table. Dependency head, its lemma and part of speech, the feature string, character offsets and the NER tag are columns of `stz_tokens`\; the former tables `stz_deps` and `stz_feats` are views over it, so every reader keeps its SQL. The lemma index is a table `stz_lemma` with the six columns of the former parquet file, physically ordered by lemma: a lookup reads only the row groups whose minimum and maximum enclose the lemma, the same pruning the parquet file gave, without a second file that can fall out of step. `stz_page_text` holds the printed text of every page, which the famulus-reference build had read from the per-document databases. No ART index is built; the tables are written in locus order and DuckDB's zone maps prune.

Full-text recall is a BM25 index @robertson2009bm25 over the sentence text and the lemmatised sentence (`lemma_text`), built with DuckDB's full-text extension. Its default tokenizer keeps only the letters a--z; the index is built with a pattern that keeps every letter and digit, so Greek and accented Latin are indexed. Compound words defeat word-level BM25: a paraphrase says "Theorie der Grenzschicht" where the source says "Grenzschichttheorie". For German and French a second index over character 4-grams of the words is kept, and its ranking is fused with the word ranking by reciprocal rank fusion.

The build reads the page documents of a collection in parallel, appends them through one writer, sorts, builds the lemma table and the two indexes, and checks gates before the swap: documents, pages, sentences and tokens equal the counts of the page documents; `stz_lemma` has one row per token with a lemma; each index has one row per indexed sentence; no sentence key repeats. A delta copies the published file forward, replaces the changed documents and rebuilds the lemma table and the indexes; above about a tenth of the documents a full build is cheaper and leaves no free blocks behind.

Two defects of the old pipeline were found on the way and fixed at their source. A worker that died left `Pool.imap_unordered` waiting for its task forever; the batch now runs on a process pool that reports a dead worker, re-runs the jobs that were in flight one by one, and records a page that kills a worker twice as an error with its name. The page that killed workers on 20 October was a subject index whose text had no sentence punctuation: Stanza received a "sentence" of 5,525 words, whose dependency parse needs memory quadratic in its length. Sentence segmentation now cuts any unit above 200 words at word boundaries into near-equal pieces.

= The famulus file
<the-famulus-file>
`<c>_famulus.duckdb` holds the tables of the FQL sidecar (`fq_*`), the evidence space (`fx_*`), the sentence records and their normal form (`sr_*`, `fnf_*`), the dossier projection (`do_*`), the famulus reference (`fr_*`) and the correspondence tables of the collection's main database (`letters_resolved`, the `ent_*` projection of the register). Its builders stay what they were; each keeps its private build state where it needs one (the FQL build keeps the cache of its language-model answers in a work file that never leaves the machine) and hands its tables over: the tables of the handed-over file replace the same-named tables of the famulus file, the other builders' tables stay untouched, scalar macros come along and the BM25 indexes of the handed-over tables are rebuilt. The defect of the evidence tables disappears with this: they are no longer written into a file that another builder replaces.

The merge of the existing stores into the famulus file checks every copied table's row count against its source and leaves behind only tables no code reads (#ref(<tbl-zusammenfuehrung>, supplement: [Table])). A table that existed in two sources would have been recorded as a conflict; none did.

Which file a role of a collection resolves to is decided in one place, `ai_lemma_index.stores`, from a recorded state per collection (`legacy` or `migrated`, separately for the corpus and the famulus file), never from which files happen to exist. The readers were moved to it in nine code bases (ai-lemma-index, famulus-proof, ai-query, ai-viewer, famulus-reference, karman-manager, LWgrammar, famulus-logic and the orchestrator); a ratchet test fails when a store name is written literally again.

= Evaluation
<evaluation>
#strong[Disk.] #ref(<tbl-inventar>, supplement: [Table]) compares the volume before and after the switch of all 28 collections with a Stanza corpus. DuckDB files fell from 2,655 with 63.5 GB to 298 with 10.8 GB; the vector stores (17.6 GB) and the CoNLL-U exports are gone. One per-document database whose write-ahead log DuckDB cannot replay (pirckheimer-sec, `jahrbuch-2012`) was kept and is named in the cleanup report. Every removed file was first written to a tar archive outside the synchronised tree, read back and compared by checksum.

#strong[Parity.] #ref(<tbl-umstellung>, supplement: [Table]) lists the switch. For 23 collections every table of the old corpus file is equal in the new one --- the row count and an order-independent checksum of all columns, computed in one process --- and so is the lemma index. Four collections differed, and in each case the old file was the stale one: in eulerOO one page had been re-parsed after the old file was built; in causality the lemma parquet dated from August; and in eulerOO-IVA the old corpus held only 802 of the 1,256 pages of the Euler--Goldbach volume, because its incremental build added missing documents but never missing pages of a document already present; and waefler-prim's old file still held a document whose folder had been removed from the collection. The new file contains the 454 missing pages, 8,416 sentences that no search could find before. karman-context had no complete old corpus to compare.

#strong[Speed.] A full build of the largest corpus, karman-nachlass (1,066 documents, 8.7 million tokens), took 65 s without and 93.5 s with the German 4-gram index on a loaded machine; a delta of one document 14--17 s. #ref(<tbl-latenz>, supplement: [Table]) gives lookup latencies measured on both files before the old one was removed: the lemma table answers in the time of the parquet file (p95 3.0 against 5.3 ms on karman-nachlass) and the dependency view faster than the old table (23.7 against 49.4 ms). Feature filters through the `stz_feats` view unnest every token and are three to five times slower than the old table; filtering the token column `feats` directly runs at the old table's speed (30--45 ms against 45 ms on karman-nachlass); the sentence-record build and the usage reading of the evidence space read the column, two measurement scripts of the notebook still use the view. The shipped BM25 stage answers with a p95 of 25 ms (Latin, enestroem), 33 ms (French, hevelius), 38 ms (English) and 60 ms (German, karman-nachlass, with the 4-gram fusion).

#strong[Answers.] The 76 SQL statements recorded by a run of the FQL bench on the old stores were replayed with their recorded bindings on the new ones (#ref(<tbl-fql>, supplement: [Table])). 72 return the same rows. Two differ between two runs on the old stores as well (an unordered limit); the statement that lists the collection's FQL tables returns fewer tables, because the dead ones were left behind.

#strong[Recall without embeddings.] The vector store was the recall path for paraphrases. Before it was removed, 200 sentences (50 each in Latin, German, English and French from four collections) were paraphrased by gemini-3.8-flash, and each engine was asked to find the source sentence from its paraphrase (#ref(<tbl-paraphrase>, supplement: [Table])). The bge-m3 store @chen2024bgem3 found it in the top 50 for 0.895 of the probes; word-level BM25 for 0.795, with German at 0.62; adding the 4-gram index for German and French raises BM25 to 0.830 overall and German to 0.74. In the top 10 the loss is larger: 0.725 against 0.890. The second embedding store, EmbeddingGemma 2, had been built only for a few collections and recalled almost nothing. The vector store is not kept for the remaining difference: the 4-gram index closes half of it, and the paraphrase recall that matters for the proofs of #cite(<grasshoff2026famulusreasoning>, form: "prose") runs over lemma and register bindings, not over similarity.

= Limits
<limits>
The measurements of speed were taken while other processes parsed and indexed collections on the same machine; they are upper bounds. The paraphrase probe measures known-item retrieval of one sentence, not the recall of all sentences that express a proposition. The store of the shared concept ontology is still copied into each collection's sentence records. The answer check of ai-query delivered 11 of 14 golden questions on the new layer; for the three that failed the store queries return the same rows on the old and the new files, so the failures lie in the routing of the questions, but there is no run of these questions on the old layer from the same day to compare.

= Tables
<tables>
#figure([
#table(
  columns: (20%, 20%, 20%, 20%, 20%),
  align: (auto,right,right,right,right,),
  table.header([store], [files before], [GB before], [files after], [GB after],),
  table.hline(),
  [per-document index DBs], [2,552], [29.61], [224], [0.26],
  [corpus/1 (`_stanza.duckdb`)], [28], [23.11], [0], [0.00],
  [lemma index (parquet)], [28], [0.12], [0], [0.00],
  [vector store (ChromaDB)], [267], [17.59], [0], [0.00],
  [corpus/2 (`_corpus.duckdb`)], [3], [1.31], [28], [4.24],
  [FQL sidecar], [4], [1.86], [0], [0.00],
  [FQL work file (build state)], [4], [0.29], [4], [0.29],
  [evidence space (`_fx`)], [9], [3.56], [0], [0.00],
  [sentence records (`_satz`)], [6], [2.51], [0], [0.00],
  [dossier projection], [7], [0.04], [0], [0.00],
  [famulus file], [10], [0.34], [15], [5.74],
  [main DB], [5], [0.08], [1], [0.01],
  [LWgrammar usage store], [13], [0.15], [13], [0.15],
  [concept cache], [10], [0.02], [10], [0.02],
  [CoNLL-U export], [145,656], [3.18], [510], [0.04],
  [Stanza page documents], [145,656], [18.97], [145,660], [18.95],
  [#strong[DuckDB files in total]], [#strong[2,655]], [#strong[63.53]], [#strong[298]], [#strong[10.84]],
)
], caption: figure.caption(
position: top, 
[
Stores on the collections volume, measured 2026-10-20T13:23 and 2026-10-20T17:32. GB = 10^9 bytes.
]), 
kind: "quarto-float-tbl", 
supplement: "Table", 
)
<tbl-inventar>


#figure([
#table(
  columns: 5,
  align: (auto,right,right,right,auto,),
  table.header([collection], [corpus/1 MB], [corpus/2 MB], [build s], [parity],),
  table.hline(),
  [braheOO], [1,152], [181], [20.9], [equal],
  [causality], [91], [18], [3.3], [differs (a)],
  [classical-philology], [7], [5], [0.8], [equal],
  [copernicus-prim], [927], [124], [17.7], [equal],
  [copernicus-sec], [1,812], [283], [45.9], [equal],
  [enestroem], [2,897], [514], [74.3], [equal],
  [eulerOO-IVA], [1,241], [207], [17.0], [differs (b)],
  [eulerOO], [1,287], [217], [23.7], [differs (c)],
  [grasshoff], [528], [73], [14.3], [equal],
  [greek-astronomy], [42], [7], [2.2], [equal],
  [hartmann], [23], [5], [1.9], [equal],
  [hevelius], [500], [58], [9.2], [equal],
  [karman-context], [17], [619], [--], [not compared (d)],
  [karman-nachlass], [4,697], [689], [93.5], [equal],
  [kepler-prim], [3,694], [525], [54.6], [equal],
  [kepler-sec], [1,223], [194], [15.3], [equal],
  [kepleriana-munich-2024], [33], [6], [1.3], [equal],
  [kirch], [1,113], [148], [37.6], [equal],
  [medieval-sec], [438], [73], [9.4], [equal],
  [natural-philosophy], [50], [14], [3.6], [equal],
  [neugebauer], [272], [64], [7.2], [equal],
  [pirckheimer-sec], [20], [7], [1.4], [equal],
  [pirckheimer], [103], [17], [4.0], [equal],
  [tractatus-typoskripte], [61], [16], [4.0], [equal],
  [waefler-prim], [406], [67], [9.7], [differs (e)],
  [waefler-sec], [342], [72], [14.9], [equal],
  [wittgenstein-mechanik-sec], [23], [7], [1.2], [equal],
  [wittgenstein-mechanik], [114], [28], [4.7], [equal],
)
], caption: figure.caption(
position: top, 
[
The corpus switch, 28 collections. Parity compares row count and an order-independent checksum of every corpus/1 table with corpus/2.
]), 
kind: "quarto-float-tbl", 
supplement: "Table", 
)
<tbl-umstellung>


#block[
#set enum(numbering: "(a)", start: 1)
+ causality: only lemma\_index.parquet differed: it dated from 2 Aug 2026, the corpus/1 tokens from 20 Oct; stz\_lemma equals the tokens \
+ eulerOO-IVA: corpus/1 held 802 of the 1,256 pages of 978-3-0348-0892-7 (IVA 4): its incremental build adds missing documents, never missing pages of a present document; corpus/2 has all 1,256 (+454 pages, +8,416 sentences) \
+ eulerOO: one page (978-3-7643-0868-1, p.~459) was re-parsed after corpus/1 was built; corpus/2 reproduces the current page document \
+ karman-context: no comparison: corpus/1 held 1 of 77 documents (its per-document ingest never finished); corpus/2 passed its gates against the page documents \
+ waefler-prim: corpus/1 still held document Waefler2001 (5 pages), whose folder no longer exists in the collection; corpus/2 holds exactly the documents on disk
]

#figure([
#table(
  columns: (16.67%, 16.67%, 16.67%, 16.67%, 16.67%, 16.67%),
  align: (auto,auto,right,right,right,right,),
  table.header([collection], [stores merged], [MB before], [MB after], [tables], [left behind],),
  table.hline(),
  [braheOO], [fr + fx], [388], [100], [27], [1],
  [copernicus-prim], [fr + fx + satz], [660], [370], [55], [2],
  [enestroem], [fr + fx + satz + dossier], [1,827], [1,078], [65], [4],
  [eulerOO-IVA], [fr + fql + satz + dossier], [464], [322], [73], [6],
  [eulerOO], [fr + fx + satz + dossier], [1,001], [538], [65], [4],
  [hartmann], [fr], [4], [4], [14], [0],
  [hevelius], [fr + main + fql + satz + dossier], [383], [235], [116], [5],
  [karman-context], [fr + fx + satz + dossier], [1,263], [554], [65], [4],
  [karman-nachlass], [fr + main + fql + dossier], [1,714], [853], [66], [4],
  [kepler-prim], [fr + fx], [987], [228], [13], [1],
  [kepler-sec], [fr + fx + satz], [911], [652], [55], [2],
  [kepleriana-munich-2024], [fr + main], [5], [4], [11], [0],
  [pirckheimer], [fr + main + fql + dossier], [27], [18], [52], [3],
  [waefler-prim], [fr + fx + satz], [390], [220], [41], [2],
  [waefler-sec], [fr + fx], [99], [38], [12], [1],
)
], caption: figure.caption(
position: top, 
[
The derived stores merged into one famulus file per collection; #emph[left behind] counts tables no code reads (`do_nennung`, `do_querverweis`, `fq_segments`, `fq_tag_topic`, `fx_formen`, `sr_concept_up`).
]), 
kind: "quarto-float-tbl", 
supplement: "Table", 
)
<tbl-zusammenfuehrung>


#figure([
#table(
  columns: (16.67%, 16.67%, 16.67%, 16.67%, 16.67%, 16.67%),
  align: (auto,right,right,right,right,right,),
  table.header([collection], [lemma p95 parquet], [lemma p95 stz\_lemma], [SVO p95 old], [SVO p95 view], [BM25 p95],),
  table.hline(),
  [hevelius], [2.11], [2.17], [4.69], [4.82], [12.57],
  [karman-nachlass], [5.31], [3.01], [49.41], [23.71], [83.29],
)
], caption: figure.caption(
position: top, 
[
Query latency in ms (p95 over 100 lemmas stratified by frequency, 20 dependency heads, 30 BM25 queries; 5 runs each after a warm-up), on both files before the old one was removed; BM25 over words only, before the 4-gram index was added.
]), 
kind: "quarto-float-tbl", 
supplement: "Table", 
)
<tbl-latenz>


#figure([
#table(
  columns: (14.29%, 14.29%, 14.29%, 14.29%, 14.29%, 14.29%, 14.29%),
  align: (auto,right,right,right,right,right,right,),
  table.header([engine], [recall\@10], [recall\@50], [la], [de], [en], [fr],),
  table.hline(),
  [vector store, bge-m3 (retired)], [0.890], [0.895], [1.00], [0.86], [0.90], [0.82],
  [vector store, EmbeddingGemma 2 (retired)], [0.085], [0.085], [0.00], [0.14], [0.06], [0.14],
  [BM25, words], [0.725], [0.795], [0.96], [0.62], [0.84], [0.76],
  [BM25, lemmata], [0.560], [0.645], [0.54], [0.46], [0.82], [0.76],
  [BM25 words + 4-grams (de, fr), shipped], [0.725], [0.830], [0.94], [0.74], [0.84], [0.80],
)
], caption: figure.caption(
position: top, 
[
Known-item retrieval of 200 source sentences from paraphrases written by gemini-3.8-flash (50 per language; per-language columns are recall\@50).
]), 
kind: "quarto-float-tbl", 
supplement: "Table", 
)
<tbl-paraphrase>


#figure([
#table(
  columns: (20%, 20%, 20%, 20%, 20%),
  align: (right,right,right,right,right,),
  table.header([statements], [identical rows], [different], [median SQL ms before], [after],),
  table.hline(),
  [76], [72], [4], [20.8], [22.9],
)
], caption: figure.caption(
position: top, 
[
The 76 SQL statements of the FQL bench, replayed with their recorded bindings on the old stores and on corpus/2 + the famulus file. Differing statements: B1-person-register (eulerOO-IVA), F4-zeitreihe-perioden (eulerOO-IVA), I2-faehigkeiten (eulerOO-IVA), I2-faehigkeiten (karman-nachlass).
]), 
kind: "quarto-float-tbl", 
supplement: "Table", 
)
<tbl-fql>





#bibliography(("bibliography.bib"))