Initial commit: Padre Matos Soares (1932) Bible corpus and pms CLIHEADmaster

2 files changed, 791 insertions, 0 deletions

diff --git a/scripts/extract_pms.py b/scripts/extract_pms.py
new file mode 100755
index 0000000..22aa1e3
--- /dev/null
+++ b/scripts/extract_pms.py
@@ -0,0 +1,640 @@
+#!/usr/bin/env python3
+"""
+Extract the Padre Matos Soares (1932) Catholic Bible from pms.pdf into:
+  - data/pms.tsv       (BookName, BookAbbr, BookNum, Chapter, Verse, Text)
+  - data/pms_notes.tsv (BookNum, Chapter, VerseStart, VerseEnd, Label, NoteText)
+  - pms.txt            (Clean, human-readable formatted text of the Bible + notes)
+"""
+
+import os
+import re
+import subprocess
+import sys
+from lxml import etree
+
+REPO_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PDF_PATH = os.path.join(REPO_DIR, "pms.pdf")
+BBOX_CACHE = "/tmp/pms_bbox.html"
+TSV_PATH = os.path.join(REPO_DIR, "data", "pms.tsv")
+NOTES_TSV_PATH = os.path.join(REPO_DIR, "data", "pms_notes.tsv")
+TXT_PATH = os.path.join(REPO_DIR, "pms.txt")
+
+BOOKS = [
+    (1, "Génesis", "Gn", 18, 115, 50),
+    (2, "Êxodo", "Ex", 116, 193, 40),
+    (3, "Levítico", "Lv", 194, 245, 27),
+    (4, "Números", "Nm", 246, 319, 36),
+    (5, "Deuteronómio", "Dt", 320, 387, 34),
+    (6, "Josué", "Js", 398, 441, 24),
+    (7, "Juízes", "Jz", 442, 489, 21),
+    (8, "Rut", "Rt", 490, 497, 4),
+    (9, "1 Reis", "1Rs", 498, 561, 31),
+    (10, "2 Reis", "2Rs", 562, 611, 24),
+    (11, "3 Reis", "3Rs", 612, 667, 22),
+    (12, "4 Reis", "4Rs", 668, 719, 25),
+    (13, "1 Paralipómenos", "1Par", 720, 769, 29),
+    (14, "2 Paralipómenos", "2Par", 770, 831, 36),
+    (15, "1 Esdras", "1Esd", 832, 850, 10),
+    (16, "2 Esdras (Neemias)", "2Esd", 851, 876, 13),
+    (17, "Tobias", "Tb", 877, 894, 14),
+    (18, "Judit", "Jt", 895, 918, 16),
+    (19, "Ester", "Est", 919, 941, 16),
+    (20, "Job", "Job", 942, 1013, 42),
+    (21, "Salmos", "Sl", 1019, 1196, 150),
+    (22, "Provérbios", "Pr", 1197, 1255, 31),
+    (23, "Eclesiastes", "Ecl", 1256, 1272, 12),
+    (24, "Cântico dos Cânticos", "Ct", 1273, 1287, 8),
+    (25, "Sabedoria", "Sb", 1288, 1329, 19),
+    (26, "Eclesiástico", "Eclo", 1330, 1439, 51),
+    (27, "Isaías", "Is", 1440, 1543, 66),
+    (28, "Jeremias", "Jr", 1544, 1647, 52),
+    (29, "Lamentações", "Lm", 1648, 1661, 5),
+    (30, "Baruc", "Br", 1662, 1675, 6),
+    (31, "Ezequiel", "Ez", 1676, 1774, 48),
+    (32, "Daniel", "Dn", 1775, 1816, 14),
+    (33, "Oséas", "Os", 1817, 1833, 14),
+    (34, "Joel", "Jl", 1834, 1839, 3),
+    (35, "Amós", "Am", 1840, 1851, 9),
+    (36, "Abdias", "Abd", 1852, 1853, 1),
+    (37, "Jonas", "Jn", 1854, 1857, 4),
+    (38, "Miquéas", "Mq", 1858, 1866, 7),
+    (39, "Naum", "Na", 1867, 1870, 3),
+    (40, "Habacuc", "Hab", 1871, 1875, 3),
+    (41, "Sofonias", "Sf", 1876, 1880, 3),
+    (42, "Ageu", "Ag", 1881, 1884, 2),
+    (43, "Zacarias", "Zc", 1885, 1902, 14),
+    (44, "Malaquias", "Ml", 1903, 1908, 4),
+    (45, "1 Macabeus", "1Mc", 1909, 1967, 16),
+    (46, "2 Macabeus", "2Mc", 1968, 2010, 15),
+    (47, "S. Mateus", "Mt", 2024, 2101, 28),
+    (48, "S. Marcos", "Mc", 2102, 2145, 16),
+    (49, "S. Lucas", "Lc", 2146, 2219, 24),
+    (50, "S. João", "Jo", 2220, 2273, 21),
+    (51, "Actos dos Apóstolos", "At", 2274, 2335, 28),
+    (52, "Romanos", "Rm", 2336, 2365, 16),
+    (53, "1 Coríntios", "1Cor", 2366, 2393, 16),
+    (54, "2 Coríntios", "2Cor", 2394, 2411, 13),
+    (55, "Gálatas", "Gl", 2412, 2421, 6),
+    (56, "Efésios", "Ef", 2422, 2431, 6),
+    (57, "Filipenses", "Fp", 2432, 2439, 4),
+    (58, "Colossenses", "Cl", 2440, 2446, 4),
+    (59, "1 Tessalonicenses", "1Ts", 2447, 2453, 5),
+    (60, "2 Tessalonicenses", "2Ts", 2454, 2457, 3),
+    (61, "1 Timóteo", "1Tm", 2458, 2465, 6),
+    (62, "2 Timóteo", "2Tm", 2466, 2471, 4),
+    (63, "Tito", "Tt", 2472, 2474, 3),
+    (64, "Filémon", "Fm", 2475, 2476, 1),
+    (65, "Hebreus", "Hb", 2477, 2498, 13),
+    (66, "S. Tiago", "Tg", 2499, 2506, 5),
+    (67, "1 S. Pedro", "1Pd", 2507, 2514, 5),
+    (68, "2 S. Pedro", "2Pd", 2515, 2520, 3),
+    (69, "1 S. João", "1Jo", 2521, 2528, 5),
+    (70, "2 S. João", "2Jo", 2529, 2530, 1),
+    (71, "3 S. João", "3Jo", 2530, 2531, 1),
+    (72, "S. Judas", "Jd", 2532, 2534, 1),
+    (73, "Apocalipse", "Ap", 2535, 2568, 22),
+]
+
+# Pages 547 and 548 in pms.pdf are a duplicate scan of pages 549 and 550 (1 Reis 25:1-28)
+DUP_PAGES = {547, 548}
+
+ROMAN_VAL = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100}
+
+def parse_roman(s):
+    if re.match(r"^[UÚ]\s*N\s*I\s*C\s*O$", s.strip(), re.I) or "NICO" in s.upper():
+        return 1
+    s = (s.replace("Í", "I").replace("í", "I").replace("J", "I").replace("j", "I")
+          .replace("1", "I").replace("0", "O").replace("!", "I").replace("l", "I")
+          .replace("f", "I").replace("i", "I").upper().strip())
+    s = re.sub(r"[^IVXLC]", "", s)
+    if not s:
+        return None
+    total = 0
+    prev = 0
+    for ch in reversed(s):
+        v = ROMAN_VAL.get(ch, 0)
+        if v < prev:
+            total -= v
+        else:
+            total += v
+            prev = v
+    return total
+
+CAP_EXTRACT_RE = re.compile(
+    r"^(?:(?:\S+\s+){0,2}?)?[\^.\s]*(?:[CGÇ]\s*[aáàâAÁÀÂ\\*°?]*\s*(?:[\*\\?35]+\s*)?[pP35\\.°*\\?]+|[aáàâAÁÀÂ]\s*[pP]\s*\.|[pP]\s*\.?\s+(?=[IVXLC]))\s*[.i,°*\s]*([IVXLC0-9ÍíJj!lfi\s]+?)\s*(?:[.,]?\s*[—\-–■\'\^~]+(?:\s*t\s*[—\-–■\'\^~]*)?|\s+1\b)"
+)
+
+FOOT_CAP_RE = re.compile(
+    r"^\s*(?:[COoc]\s*[Aaa]\s*[Pp]\s*[,.\-]?|S\s*\.)\s*([IVXLCivxlc0-9ÍíJj!lfi\s]+?)\.?\s*$"
+)
+FOOT_NOTE_RE = re.compile(
+    r"^\s*(\d+(?:\s*[-–,]\s*\d+|\s+e\s+segs?\.)*)\s*[.,\-]\s+(\S.*)"
+)
+
+SECTION_HDR_RE = re.compile(
+    r"^\s*(?:"
+    r"(?:PRIMEIRA|SEGUNDA|SECUNDA|TERCEIRA|QUARTA|QUINTA|SEXTA|SÉTIMA)\s+PARTE\b|"
+    r"FIM\s+D[OAE]\s+(?:LIVRO|PROFECIA|EPÍSTOLA|EVANGELHO|SEGUNDO|PRIMEIRO)\b|"
+    r"P\s*R\s*E\s*[ÂA]\s*M\s*B\s*U\s*L\s*O\s*$|"
+    r"P\s*R\s*[ÓO]\s*L\s*O\s*G\s*O\s*$|"
+    r"E\s*P\s*[ÍI]\s*L\s*O\s*G\s*O\s*$|"
+    r"C\s*O\s*N\s*C\s*L\s*U\s*S\s*[ÃA]\s*O\s*$|"
+    r"CANTO\s+[IVX]+\b|"
+    r"[IVX]+\.?\s*[—\-–]\s+\S|"
+    r"§\s*\d+"
+    r")",
+    re.I
+)
+
+ENCLITICS = {
+    "se", "me", "te", "lhe", "lhes", "nos", "vos",
+    "o", "a", "os", "as", "lo", "la", "los", "las",
+    "no", "na", "nas", "lho", "lha", "lhos", "lhas",
+    "mo", "ma", "mos", "mas", "to", "ta", "tos", "tas",
+}
+
+SINGLE_LETTER_WORDS = {"a", "e", "o", "à", "á", "é", "ó", "ò", "ú", "A", "E", "O", "À", "Á", "É", "Ó"}
+
+def fix_spaced_letters(text):
+    """Collapse OCR spaced-out small-caps/italics like 'O E s p irito de D e u s' or 'R e c e b a'."""
+    tokens = text.split()
+    if len(tokens) < 2:
+        return text
+    # Detect if there are runs of single-letter tokens
+    out = []
+    i = 0
+    n = len(tokens)
+    while i < n:
+        # Check if we are starting a spaced-out sequence
+        if len(tokens[i]) == 1 and tokens[i].isalpha() and i + 1 < n and len(tokens[i+1]) <= 2 and tokens[i+1].rstrip(".,;:!?").isalpha():
+            # Collect contiguous short fragments if at least 2 single-letter tokens
+            j = i
+            single_cnt = 0
+            while j < n:
+                t_clean = tokens[j].strip(".,;:!?()")
+                if len(t_clean) == 1 and t_clean.isalpha():
+                    single_cnt += 1
+                    j += 1
+                elif len(t_clean) == 2 and t_clean.isalpha() and single_cnt >= 2 and t_clean.lower() not in {"de", "do", "da", "em", "no", "na", "os", "as", "um", "ao", "se", "me", "te", "tu", "eu", "já", "só", "fé"}:
+                    j += 1
+                else:
+                    break
+            if single_cnt >= 2 and (j - i) >= 3:
+                # Reconstruct words from tokens[i:j], keeping standalone 'e', 'a', 'o' when appropriate
+                buf = []
+                for k in range(i, j):
+                    tk = tokens[k]
+                    buf.append(tk)
+                    if tk[-1] in ".,;:!?)":
+                        out.append("".join(buf))
+                        buf = []
+                if buf:
+                    out.append("".join(buf))
+                i = j
+                continue
+        out.append(tokens[i])
+        i += 1
+    return " ".join(out)
+
+def join_line_words(words, h_thresh=7.0, mark_verse_gaps=False):
+    if not words:
+        return ""
+    out = [words[0][4]]
+    for prev, curr in zip(words, words[1:]):
+        gap = curr[0] - prev[2]
+        t1, t2 = prev[4], curr[4]
+        h1, h2 = (prev[3] - prev[1]), (curr[3] - curr[1])
+        c1, c2 = t1.strip(".,;:!?()"), t2.strip(".,;:!?()")
+        if t1.isdigit() and t2.isdigit() and gap <= 4.0:
+            out[-1] = out[-1] + t2
+        elif (
+            not t1.isdigit() and not t2.isdigit()
+            and not t1.endswith((".", ",", ";", ":", "!", "?"))
+            and (
+                (max(h1, h2) < h_thresh - 0.2 and gap <= 1.75 and (len(c1) <= 2 or len(c2) <= 2))
+                or (gap <= 1.15 and (len(c1) == 1 or len(c2) == 1) and c1.lower() not in {"e", "a", "o", "á", "à", "é"})
+            )
+        ):
+            out[-1] = out[-1] + t2
+        else:
+            if (
+                mark_verse_gaps
+                and gap >= 8.2
+                and t1.endswith((".", ":", ";", "!", "?", ")"))
+                and not out[-1].lower().endswith(("cap.", "ap.", "p."))
+                and t2 and (t2[0].isupper() or t2[0] in "«\"(")
+            ):
+                out.append("@@GAP@@")
+            out.append(t2)
+    return " ".join(out)
+
+def join_lines_text(lines):
+    """Join multiple lines of text, de-hyphenating line-break hyphens cleanly."""
+    if not lines:
+        return ""
+    res = ""
+    for line in lines:
+        # Strip inline footnote reference markers at end of line (e.g. "* 1", "* 1 6", "I * 8 9", "*")
+        line = re.sub(r"\s+(?:[I1l]\s+)?\*\s*(?:\d+(?:\s+\d+)*)?\s*$", "", line)
+        line = re.sub(r"\*+", "", line)
+        line = line.strip()
+        if not line:
+            continue
+        if not res:
+            res = line
+            continue
+        if res.endswith("\xad"):
+            first_next = line.split()[0].rstrip(".,;:!?()") if line.split() else ""
+            if first_next.lower() in ENCLITICS or "-" in first_next:
+                res = res[:-1] + "-" + line
+            else:
+                res = res[:-1] + line
+        elif res.endswith("-") and not res.endswith(" -") and not res.endswith("—"):
+            first_next = line.split()[0].rstrip(".,;:!?()") if line.split() else ""
+            if first_next.lower() in ENCLITICS or "-" in first_next or (first_next and first_next[0].isupper()):
+                res = res + line
+            else:
+                res = res[:-1] + line
+        else:
+            res = res + " " + line
+    # Also clean any internal soft hyphens followed by space
+    res = re.sub(r"\xad\s+", "", res)
+    res = res.replace("\xad", "")
+    # Clean stray spaces before punctuation
+    res = re.sub(r"\s+([.,;:!?])", r"\1", res)
+    res = re.sub(r"\s+", " ", res).strip()
+    return res
+
+def load_pages():
+    if not os.path.exists(BBOX_CACHE):
+        print(f"Generating {BBOX_CACHE} from {PDF_PATH}...")
+        subprocess.run(["pdftotext", "-bbox", PDF_PATH, BBOX_CACHE], check=True)
+    pages = []
+    context = etree.iterparse(BBOX_CACHE, events=("end",), tag="{http://www.w3.org/1999/xhtml}page", html=False)
+    for _, elem in context:
+        words = [
+            (float(w.attrib["xMin"]), float(w.attrib["yMin"]), float(w.attrib["xMax"]), float(w.attrib["yMax"]), w.text or "")
+            for w in elem
+        ]
+        pages.append(words)
+        elem.clear()
+    return pages
+
+def get_page_lines(pages, pno):
+    h_thresh = 6.8 if pno < 390 else (6.6 if pno < 1015 else (6.8 if pno < 2012 else 7.2))
+    words = [w for w in pages[pno-1] if w[1] >= 40.0]
+    main_w = [w for w in words if (w[3]-w[1]) >= h_thresh]
+    if not main_w:
+        return [], []
+    xs0 = sorted(w[0] for w in main_w)
+    xs1 = sorted(w[2] for w in main_w)
+    col_x0 = xs0[len(xs0)//10]
+    col_x1 = xs1[len(xs1)*9//10]
+    is_even = (pno % 2 == 0) if (1019 <= pno <= 2010) else (pno % 2 == 1)
+    col_w = [w for w in words if (w[3]-w[1]) >= 5.0 and w[1] >= 44.0]
+    if is_even:
+        valid_x0 = sorted(w[0] for w in col_w if w[0] > 48.0)
+        valid_x1 = sorted(w[2] for w in col_w)
+        raw_left = valid_x0[len(valid_x0)//30] if valid_x0 else 56.0
+        raw_right = valid_x1[len(valid_x1)*29//30] if valid_x1 else 284.0
+        page_shift = max(0.0, raw_right - 284.0)
+        main_left = min(58.5 + page_shift, raw_left)
+    else:
+        valid_x1 = [w[2] for w in col_w if w[0] < 248.0 and w[2] < 263.0]
+        raw_right = max(valid_x1) if valid_x1 else 254.0
+        main_right = max(250.0, raw_right)
+    filtered = []
+    for w in words:
+        x0, y0, x1, y1, txt = w
+        h = y1 - y0
+        if is_even:
+            if h >= h_thresh - 0.3 and txt.strip(".,").isdigit() and x0 >= main_left - 12.5:
+                filtered.append(w)
+                continue
+            if x1 <= main_left - 0.5 or x0 < main_left - 3.5:
+                continue
+        else:
+            if x0 >= main_right + 0.5 or x1 > main_right + 3.5:
+                continue
+        filtered.append(w)
+    words_sorted = sorted(filtered, key=lambda w: ((w[1]+w[3])/2.0, w[0]))
+    lines = []
+    for w in words_sorted:
+        yc = (w[1]+w[3])/2.0
+        if lines and abs(yc - lines[-1]["yc"]) <= 3.0:
+            lines[-1]["words"].append(w)
+            lines[-1]["yc"] = sum((x[1]+x[3])/2.0 for x in lines[-1]["words"]) / len(lines[-1]["words"])
+        else:
+            lines.append({"yc": yc, "words": [w]})
+    for l in lines:
+        l["words"].sort(key=lambda w: w[0])
+        hs = sorted(w[3]-w[1] for w in l["words"])
+        l["mh"] = hs[len(hs)//2]
+        l["x0"] = l["words"][0][0]
+        l["x1"] = l["words"][-1][2]
+        l["text"] = join_line_words(l["words"], h_thresh=h_thresh, mark_verse_gaps=True)
+    if lines and lines[0]["yc"] < 43.5 and re.search(r"[—,]|\b\d+\s*$|^\s*\d+\b", lines[0]["text"]):
+        if not re.search(r"^\s*\d+\s+[A-ZÁÉÍÓÚÂÊÔÃÕÇ]", lines[0]["text"]):
+            lines = lines[1:]
+    last_main_idx = -1
+    last_cap_idx = max(
+        (i for i, l in enumerate(lines) if CAP_EXTRACT_RE.search(l["text"]) or "único" in l["text"] or "ú n" in l["text"] or re.search(r"^\s*S\s*A\s*L\s*M\s*O\b", l["text"], re.I)),
+        default=-1
+    )
+    for i, l in enumerate(lines):
+        dy = (l["yc"] - lines[i-1]["yc"]) if i > 0 else 0.0
+        if i > last_cap_idx and last_main_idx >= 0 and l["yc"] > 220.0:
+            if FOOT_CAP_RE.match(l["text"]):
+                break
+            if dy >= 10.0 and FOOT_NOTE_RE.match(l["text"]):
+                break
+            if i + 1 < len(lines) and (lines[i+1]["yc"] - l["yc"]) < 6.9 and FOOT_NOTE_RE.match(l["text"]):
+                break
+        if l["mh"] >= h_thresh and len(l["words"]) >= 2:
+            last_main_idx = i
+        elif l["mh"] >= h_thresh and len(l["words"]) == 1 and not l["text"].isdigit():
+            last_main_idx = i
+        elif CAP_EXTRACT_RE.search(l["text"]):
+            last_main_idx = i
+    return lines[:last_main_idx+1], lines[last_main_idx+1:]
+
+def is_section_header(ml, i):
+    l = ml[i]
+    txt = l["text"].strip()
+    if CAP_EXTRACT_RE.match(txt) or "único" in txt.lower() or "ú n" in txt.lower():
+        return False
+    if re.match(r"^\s*S\s*A\s*L\s*M\s*O\b", txt, re.I):
+        return False
+    if re.match(r"^\s*\d+\s+[A-ZÁÉÍÓÚÂÊÔÃÕÇ«\"(a-z]", txt):
+        return False
+    if SECTION_HDR_RE.match(txt):
+        return True
+    dy_prev = (l["yc"] - ml[i-1]["yc"]) if i > 0 else 20.0
+    dy_next = (ml[i+1]["yc"] - l["yc"]) if i+1 < len(ml) else 20.0
+    if i > 0 and dy_prev >= 18.0 and dy_next >= 14.0:
+        return True
+    return False
+
+def parse_ocr_vnum(tok, cur_v):
+    t = tok.strip(".,;:()")
+    if t.isdigit():
+        v = int(t)
+        if cur_v + 1 <= v <= cur_v + 3:
+            return v
+        return None
+    t2 = (t.replace("l", "1").replace("I", "1").replace("!", "1").replace("|", "1")
+           .replace("O", "0").replace("o", "0").replace("õ", "5").replace("S", "5"))
+    if t2.isdigit() and int(t2) == cur_v + 1:
+        return cur_v + 1
+    return None
+
+def split_chapter_verses(bnum, cnum, clines):
+    """Split a chapter's lines into [(verse_num, verse_text), ...]."""
+    if bnum == 21:
+        # In Salmos, strip the italic editorial summary lines before verse 1
+        start_idx = 0
+        for idx, line in enumerate(clines):
+            if re.search(r"^\s*1\b|\b1\s+[A-ZÁÉÍÓÚÂÊÔÃÕÇ«\"(a-z]", line):
+                start_idx = idx
+                break
+            if cnum in (115, 147) and re.search(r"^\s*1[02]\b", line):
+                start_idx = idx
+                break
+            if cnum == 42 and re.search(r"Julga-me", line):
+                start_idx = idx
+                break
+            if cnum == 48 and re.search(r"Para o fim", line):
+                start_idx = idx
+                break
+        clines = clines[start_idx:]
+        if clines:
+            clines[0] = re.sub(r"^.*?\b1\s+", "", clines[0], count=1) if re.search(r"\b1\s+", clines[0]) else clines[0]
+
+    joined = join_lines_text(clines)
+    if not joined:
+        return [(1, "")]
+
+    tokens = joined.split()
+    verses = []
+    cur_v = 1
+    cur_words = []
+
+    if bnum == 21 and cnum in (115, 147):
+        first_num = 10 if cnum == 115 else 12
+        if tokens and tokens[0].strip(".,") == str(first_num):
+            tokens = tokens[1:]
+        cur_v = first_num
+
+    for idx, tok in enumerate(tokens):
+        v = parse_ocr_vnum(tok, cur_v)
+        # Ensure tok isn't part of a parenthetical reference or followed by nothing
+        if v is not None and idx + 1 < len(tokens) and cur_words:
+            if v == cur_v + 2 and "@@GAP@@" in cur_words:
+                gap_pos = cur_words.index("@@GAP@@")
+                w1 = [w for w in cur_words[:gap_pos] if w != "@@GAP@@"]
+                w2 = [w for w in cur_words[gap_pos+1:] if w != "@@GAP@@"]
+                if w1 and w2:
+                    v_out1 = (cur_v - 9) if (bnum == 21 and cnum == 115 and cur_v >= 10) else (
+                             (cur_v - 11) if (bnum == 21 and cnum == 147 and cur_v >= 12) else cur_v)
+                    verses.append((v_out1, " ".join(w1).strip()))
+                    cur_v += 1
+                    cur_words = w2
+            clean_words = [w for w in cur_words if w != "@@GAP@@"]
+            if clean_words:
+                v_out = (cur_v - 9) if (bnum == 21 and cnum == 115 and cur_v >= 10) else (
+                        (cur_v - 11) if (bnum == 21 and cnum == 147 and cur_v >= 12) else cur_v)
+                verses.append((v_out, " ".join(clean_words).strip()))
+            cur_v = v
+            cur_words = []
+        else:
+            cur_words.append(tok)
+
+    clean_words = [w for w in cur_words if w != "@@GAP@@"]
+    if clean_words or not verses:
+        v_out = (cur_v - 9) if (bnum == 21 and cnum == 115 and cur_v >= 10) else (
+                (cur_v - 11) if (bnum == 21 and cnum == 147 and cur_v >= 12) else cur_v)
+        verses.append((v_out, " ".join(clean_words).strip()))
+
+    # Fill any skipped verse numbers if v jumped by 2 so verse numbers are strictly contiguous
+    return verses
+
+def parse_book_footnotes(bnum, foot_lines_by_page, page_first_chap):
+    """Parse footnotes across a book's pages into [(chap, v_start, v_end, label, text), ...]."""
+    notes = []
+    cur_chap = 1
+    cur_note = None  # [chap, v_start, v_end, label, lines_list]
+
+    for pno, flines in foot_lines_by_page:
+        if pno in page_first_chap and cur_chap < page_first_chap[pno] - 1:
+            cur_chap = page_first_chap[pno]
+        for l in flines:
+            txt = l["text"].replace("@@GAP@@", "").strip()
+            if not txt:
+                continue
+            m_cap = FOOT_CAP_RE.match(txt)
+            if m_cap:
+                r = parse_roman(m_cap.group(1))
+                if r is not None:
+                    if cur_note:
+                        notes.append((cur_note[0], cur_note[1], cur_note[2], cur_note[3], join_lines_text(cur_note[4])))
+                        cur_note = None
+                    cur_chap = r
+                continue
+            m_note = FOOT_NOTE_RE.match(txt)
+            if m_note:
+                if cur_note:
+                    notes.append((cur_note[0], cur_note[1], cur_note[2], cur_note[3], join_lines_text(cur_note[4])))
+                label = m_note.group(1).strip()
+                rest = m_note.group(2).strip()
+                nums = [int(x) for x in re.findall(r"\d+", label)]
+                v_start = nums[0] if nums else 1
+                v_end = nums[-1] if len(nums) > 1 else v_start
+                if v_end < v_start:
+                    v_end = v_start
+                cur_note = [cur_chap, v_start, v_end, label, [rest]]
+            else:
+                if cur_note:
+                    cur_note[4].append(txt)
+
+    if cur_note:
+        notes.append((cur_note[0], cur_note[1], cur_note[2], cur_note[3], join_lines_text(cur_note[4])))
+    return [n for n in notes if n[4]]
+
+def main():
+    os.makedirs(os.path.dirname(TSV_PATH), exist_ok=True)
+    pages = load_pages()
+
+    all_verses = []
+    all_notes = []
+
+    for bnum, bname, babbr, p1, p2, exp_ch in BOOKS:
+        book_lines = []
+        foot_lines_by_page = []
+        page_first_chap = {}
+
+        for pno in range(p1, p2+1):
+            if pno in DUP_PAGES:
+                continue
+            ml, fl = get_page_lines(pages, pno)
+            if bnum == 70 and pno == 2530:
+                ml = [l for l in ml if l["yc"] < 150.0]
+                fl = []
+            elif bnum == 71 and pno == 2530:
+                ml = [l for l in ml if l["yc"] >= 150.0]
+            for i, l in enumerate(ml):
+                if is_section_header(ml, i):
+                    continue
+                book_lines.append((pno, l["yc"], l["mh"], l["text"]))
+            foot_lines_by_page.append((pno, fl))
+
+        chapters = []
+        for pno, yc, mh, txt in book_lines:
+            is_ch = False
+            if bnum == 21:
+                m = re.search(r"^\s*S\s*A\s*L\s*M\s*O\s+([IVXLC0-9ÍíJjL!lfi\s]+)\.?\s*$", txt, re.I)
+                if m:
+                    cnum = len(chapters) + 1
+                    chapters.append((cnum, []))
+                    page_first_chap.setdefault(pno, cnum)
+                    is_ch = True
+            elif exp_ch == 1:
+                m = re.match(
+                    r"^(?:(?:\S+\s+){0,2}?)?[\^.\s]*[CGÇ]\s*[aáàâAÁÀÂ]*\s*[pP]\s*\.?\s*[uúUÚ]\s*n\s*i?\s*c\s*o\s*\.?\s*[—\-–]+\s*(?:1\b\s*)?(.*)",
+                    txt, re.I
+                )
+                if m and not chapters:
+                    chapters.append((1, [m.group(1)]))
+                    page_first_chap.setdefault(pno, 1)
+                    is_ch = True
+            else:
+                m = CAP_EXTRACT_RE.match(txt)
+                if m:
+                    rest = txt[m.end():].strip()
+                    rest = re.sub(r"^(?:ALEF\.\s*)?1\b\s*", "", rest, flags=re.I)
+                    cnum = len(chapters) + 1
+                    chapters.append((cnum, [rest]))
+                    page_first_chap.setdefault(pno, cnum)
+                    is_ch = True
+                elif re.match(r"^\s*CAP\.\s+1\s+(.*)", txt):
+                    m2 = re.match(r"^\s*CAP\.\s+1\s+(.*)", txt)
+                    cnum = len(chapters) + 1
+                    chapters.append((cnum, [m2.group(1)]))
+                    page_first_chap.setdefault(pno, cnum)
+                    is_ch = True
+                else:
+                    m2 = re.match(r"^\s*\.?\s*([IVXLC]+)\s*[—\-–■\^~]+\s*1\b\s*(.*)", txt)
+                    if m2:
+                        cnum = len(chapters) + 1
+                        chapters.append((cnum, [m2.group(2)]))
+                        page_first_chap.setdefault(pno, cnum)
+                        is_ch = True
+            if not is_ch and chapters:
+                chapters[-1][1].append(txt)
+
+        if len(chapters) != exp_ch:
+            print(f"WARNING: Book {bnum} ({bname}) chapters {len(chapters)} != {exp_ch}", file=sys.stderr)
+
+        for cnum, clines in chapters:
+            vlist = split_chapter_verses(bnum, cnum, clines)
+            for vnum, vtext in vlist:
+                vtext_clean = vtext.replace("\t", " ").strip()
+                if vtext_clean:
+                    all_verses.append((bname, babbr, bnum, cnum, vnum, vtext_clean))
+
+        bnotes = parse_book_footnotes(bnum, foot_lines_by_page, page_first_chap)
+        for cnum, vs, ve, label, ntext in bnotes:
+            ntext_clean = ntext.replace("\t", " ").strip()
+            if ntext_clean:
+                all_notes.append((bnum, cnum, vs, ve, label, ntext_clean))
+
+    # Write data/pms.tsv
+    with open(TSV_PATH, "w", encoding="utf-8") as f:
+        for bname, babbr, bnum, cnum, vnum, vtext in all_verses:
+            f.write(f"{bname}\t{babbr}\t{bnum}\t{cnum}\t{vnum}\t{vtext}\n")
+
+    # Write data/pms_notes.tsv
+    with open(NOTES_TSV_PATH, "w", encoding="utf-8") as f:
+        for bnum, cnum, vs, ve, label, ntext in all_notes:
+            f.write(f"{bnum}\t{cnum}\t{vs}\t{ve}\t{label}\t{ntext}\n")
+
+    # Index notes by (bnum, cnum) for pms.txt
+    notes_by_bc = {}
+    for bnum, cnum, vs, ve, label, ntext in all_notes:
+        notes_by_bc.setdefault((bnum, cnum), []).append((label, ntext))
+
+    # Write clean, human-readable pms.txt
+    with open(TXT_PATH, "w", encoding="utf-8") as f:
+        f.write("BÍBLIA SAGRADA\n")
+        f.write("Tradução e Comentários do Padre Manuel de Matos Soares (1932)\n")
+        f.write("=============================================================\n\n")
+        cur_b = None
+        cur_c = None
+        for bname, babbr, bnum, cnum, vnum, vtext in all_verses:
+            if bnum != cur_b:
+                if cur_b is not None and (cur_b, cur_c) in notes_by_bc:
+                    f.write("\n  [Notas — Capítulo " + str(cur_c) + "]\n")
+                    for lbl, ntxt in notes_by_bc[(cur_b, cur_c)]:
+                        f.write(f"  * {lbl}. {ntxt}\n")
+                cur_b = bnum
+                cur_c = None
+                f.write(f"\n\n=============================================================\n")
+                f.write(f"LIVRO {bnum}: {bname.upper()} ({babbr})\n")
+                f.write(f"=============================================================\n")
+            if cnum != cur_c:
+                if cur_c is not None and (bnum, cur_c) in notes_by_bc:
+                    f.write("\n  [Notas — Capítulo " + str(cur_c) + "]\n")
+                    for lbl, ntxt in notes_by_bc[(bnum, cur_c)]:
+                        f.write(f"  * {lbl}. {ntxt}\n")
+                cur_c = cnum
+                hdr_label = "Salmo" if bnum == 21 else "Capítulo"
+                f.write(f"\n--- {bname} — {hdr_label} {cnum} ---\n\n")
+            f.write(f"{cnum}:{vnum} {vtext}\n")
+        if cur_b is not None and (cur_b, cur_c) in notes_by_bc:
+            f.write("\n  [Notas — Capítulo " + str(cur_c) + "]\n")
+            for lbl, ntxt in notes_by_bc[(cur_b, cur_c)]:
+                f.write(f"  * {lbl}. {ntxt}\n")
+
+    print(f"Successfully extracted {len(all_verses)} verses and {len(all_notes)} commentary notes across {len(BOOKS)} books.")
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/generate_c_data.py b/scripts/generate_c_data.py
new file mode 100644
index 0000000..3afad0e
--- /dev/null
+++ b/scripts/generate_c_data.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""
+Generate src/pms_data.c from data/pms.tsv and data/pms_notes.tsv.
+"""
+
+import os
+
+REPO_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+TSV_PATH = os.path.join(REPO_DIR, "data", "pms.tsv")
+NOTES_TSV_PATH = os.path.join(REPO_DIR, "data", "pms_notes.tsv")
+OUT_C_PATH = os.path.join(REPO_DIR, "src", "pms_data.c")
+
+BOOK_ALIASES = {
+    1: "gn,gen,genesis",
+    2: "ex,exo,exodo",
+    3: "lv,lev,levitico",
+    4: "nm,num,numeros",
+    5: "dt,deut,deuteronomio",
+    6: "js,jos,josue",
+    7: "jz,juiz,juizes",
+    8: "rt,rut,rute",
+    9: "1rs,1reis,ireis,1sm,1sam,1samuel,isamuel",
+    10: "2rs,2reis,iireis,2sm,2sam,2samuel,iisamuel",
+    11: "3rs,3reis,iiireis",
+    12: "4rs,4reis,ivreis",
+    13: "1par,1paralipomenos,iparalipomenos,1cr,1cron,1cronicas,icronicas",
+    14: "2par,2paralipomenos,iiparalipomenos,2cr,2cron,2cronicas,iicronicas",
+    15: "1esd,1esdras,iesdras,esd,esdras",
+    16: "2esd,2esdras,iiesdras,ne,nee,neemias",
+    17: "tb,tob,tobias",
+    18: "jt,jdt,judit,judite",
+    19: "est,ester",
+    20: "job,jo",
+    21: "sl,sal,salmo,salmos,ps,psalmos",
+    22: "pr,prov,proverbios",
+    23: "ecl,eclesiastes,qo,coelet",
+    24: "ct,cant,cantico,canticos,canticodoscanticos",
+    25: "sb,sab,sabedoria",
+    26: "eclo,eclesiastico,sir,siracide,bensira",
+    27: "is,isa,isaias",
+    28: "jr,jer,jeremias",
+    29: "lm,lam,lamentacoes",
+    30: "br,bar,baruc",
+    31: "ez,eze,ezeq,ezequiel",
+    32: "dn,dan,daniel",
+    33: "os,ose,oseas,oseias",
+    34: "jl,joel",
+    35: "am,amos",
+    36: "abd,ab,abdias,ob,obadias",
+    37: "jn,jon,jonas",
+    38: "mq,miq,miqueas,miqueias",
+    39: "na,nau,naum",
+    40: "hab,hc,habacuc,habacuque",
+    41: "sf,sof,sofonias",
+    42: "ag,ageu",
+    43: "zc,zac,zacarias",
+    44: "ml,mal,malaquias",
+    45: "1mc,1mac,1macabeus,imacabeus",
+    46: "2mc,2mac,2macabeus,iimacabeus",
+    47: "mt,mat,mateus,smateus,saomateus",
+    48: "mc,mar,marcos,smarcos,saomarcos",
+    49: "lc,luc,lucas,slucas,saolucas",
+    50: "jo,joao,sjoao,saojoao",
+    51: "at,act,atos,actos,actosdosapostolos,atosdosapostolos",
+    52: "rm,rom,romanos",
+    53: "1cor,1co,1corintios,icorintios",
+    54: "2cor,2co,2corintios,iicorintios",
+    55: "gl,gal,galatas",
+    56: "ef,efe,efesios",
+    57: "fp,flp,fil,filipenses",
+    58: "cl,col,colossenses",
+    59: "1ts,1tes,1tessalonicenses,itessalonicenses",
+    60: "2ts,2tes,2tessalonicenses,iitessalonicenses",
+    61: "1tm,1tim,1timoteo,itimoteo",
+    62: "2tm,2tim,2timoteo,iitimoteo",
+    63: "tt,tit,tito",
+    64: "fm,flm,filemon",
+    65: "hb,heb,hebreus",
+    66: "tg,tiago,stiago,saotiago",
+    67: "1pd,1pe,1ped,1pedro,1spedro,ipedro",
+    68: "2pd,2pe,2ped,2pedro,2spedro,iipedro",
+    69: "1jo,1joao,1sjoao,ijoao",
+    70: "2jo,2joao,2sjoao,iijoao",
+    71: "3jo,3joao,3sjoao,iiijoao",
+    72: "jd,jud,judas,sjudas,saojudas",
+    73: "ap,apoc,apocalipse",
+}
+
+def c_escape(s):
+    return s.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n").replace("\r", "")
+
+def main():
+    books = {}
+    verses = []
+    with open(TSV_PATH, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.rstrip("\n")
+            if not line:
+                continue
+            parts = line.split("\t")
+            if len(parts) < 6:
+                continue
+            bname, babbr, bnum_s, cnum_s, vnum_s, vtext = parts[:6]
+            bnum = int(bnum_s)
+            cnum = int(cnum_s)
+            vnum = int(vnum_s)
+            if bnum not in books:
+                books[bnum] = (bname, babbr)
+            verses.append((bnum, cnum, vnum, vtext))
+
+    notes = []
+    if os.path.exists(NOTES_TSV_PATH):
+        with open(NOTES_TSV_PATH, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.rstrip("\n")
+                if not line:
+                    continue
+                parts = line.split("\t")
+                if len(parts) < 6:
+                    continue
+                bnum_s, cnum_s, vs_s, ve_s, label, ntext = parts[:6]
+                notes.append((int(bnum_s), int(cnum_s), int(vs_s), int(ve_s), label, ntext))
+
+    with open(OUT_C_PATH, "w", encoding="utf-8") as out:
+        out.write('/* Auto-generated from data/pms.tsv and data/pms_notes.tsv. Do not edit manually. */\n')
+        out.write('#include "pms_data.h"\n\n')
+
+        out.write("pms_book pms_books[] = {\n")
+        for bnum in sorted(books.keys()):
+            bname, babbr = books[bnum]
+            aliases = BOOK_ALIASES.get(bnum, babbr.lower())
+            out.write(f'    {{{bnum}, "{c_escape(bname)}", "{c_escape(babbr)}", "{c_escape(aliases)}"}},\n')
+        out.write("};\n")
+        out.write(f"int pms_books_length = {len(books)};\n\n")
+
+        out.write("pms_verse pms_verses[] = {\n")
+        for bnum, cnum, vnum, vtext in verses:
+            out.write(f'    {{{bnum}, {cnum}, {vnum}, "{c_escape(vtext)}"}},\n')
+        out.write("};\n")
+        out.write(f"int pms_verses_length = {len(verses)};\n\n")
+
+        out.write("pms_note pms_notes[] = {\n")
+        for bnum, cnum, vs, ve, label, ntext in notes:
+            out.write(f'    {{{bnum}, {cnum}, {vs}, {ve}, "{c_escape(label)}", "{c_escape(ntext)}"}},\n')
+        out.write("};\n")
+        out.write(f"int pms_notes_length = {len(notes)};\n")
+
+    print(f"Generated {OUT_C_PATH} ({len(books)} books, {len(verses)} verses, {len(notes)} notes).")
+
+if __name__ == "__main__":
+    main()