path: root/scripts/extract_pms.py

#!/usr/bin/env python3
"""
Extract the Padre Matos Soares (1932) Catholic Bible from pms.pdf into:
  - data/pms.tsv       (BookName, BookAbbr, BookNum, Chapter, Verse, Text)
  - data/pms_notes.tsv (BookNum, Chapter, VerseStart, VerseEnd, Label, NoteText)
  - pms.txt            (Clean, human-readable formatted text of the Bible + notes)
"""

import os
import re
import subprocess
import sys
from lxml import etree

REPO_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PDF_PATH = os.path.join(REPO_DIR, "pms.pdf")
BBOX_CACHE = "/tmp/pms_bbox.html"
TSV_PATH = os.path.join(REPO_DIR, "data", "pms.tsv")
NOTES_TSV_PATH = os.path.join(REPO_DIR, "data", "pms_notes.tsv")
TXT_PATH = os.path.join(REPO_DIR, "pms.txt")

BOOKS = [
    (1, "Génesis", "Gn", 18, 115, 50),
    (2, "Êxodo", "Ex", 116, 193, 40),
    (3, "Levítico", "Lv", 194, 245, 27),
    (4, "Números", "Nm", 246, 319, 36),
    (5, "Deuteronómio", "Dt", 320, 387, 34),
    (6, "Josué", "Js", 398, 441, 24),
    (7, "Juízes", "Jz", 442, 489, 21),
    (8, "Rut", "Rt", 490, 497, 4),
    (9, "1 Reis", "1Rs", 498, 561, 31),
    (10, "2 Reis", "2Rs", 562, 611, 24),
    (11, "3 Reis", "3Rs", 612, 667, 22),
    (12, "4 Reis", "4Rs", 668, 719, 25),
    (13, "1 Paralipómenos", "1Par", 720, 769, 29),
    (14, "2 Paralipómenos", "2Par", 770, 831, 36),
    (15, "1 Esdras", "1Esd", 832, 850, 10),
    (16, "2 Esdras (Neemias)", "2Esd", 851, 876, 13),
    (17, "Tobias", "Tb", 877, 894, 14),
    (18, "Judit", "Jt", 895, 918, 16),
    (19, "Ester", "Est", 919, 941, 16),
    (20, "Job", "Job", 942, 1013, 42),
    (21, "Salmos", "Sl", 1019, 1196, 150),
    (22, "Provérbios", "Pr", 1197, 1255, 31),
    (23, "Eclesiastes", "Ecl", 1256, 1272, 12),
    (24, "Cântico dos Cânticos", "Ct", 1273, 1287, 8),
    (25, "Sabedoria", "Sb", 1288, 1329, 19),
    (26, "Eclesiástico", "Eclo", 1330, 1439, 51),
    (27, "Isaías", "Is", 1440, 1543, 66),
    (28, "Jeremias", "Jr", 1544, 1647, 52),
    (29, "Lamentações", "Lm", 1648, 1661, 5),
    (30, "Baruc", "Br", 1662, 1675, 6),
    (31, "Ezequiel", "Ez", 1676, 1774, 48),
    (32, "Daniel", "Dn", 1775, 1816, 14),
    (33, "Oséas", "Os", 1817, 1833, 14),
    (34, "Joel", "Jl", 1834, 1839, 3),
    (35, "Amós", "Am", 1840, 1851, 9),
    (36, "Abdias", "Abd", 1852, 1853, 1),
    (37, "Jonas", "Jn", 1854, 1857, 4),
    (38, "Miquéas", "Mq", 1858, 1866, 7),
    (39, "Naum", "Na", 1867, 1870, 3),
    (40, "Habacuc", "Hab", 1871, 1875, 3),
    (41, "Sofonias", "Sf", 1876, 1880, 3),
    (42, "Ageu", "Ag", 1881, 1884, 2),
    (43, "Zacarias", "Zc", 1885, 1902, 14),
    (44, "Malaquias", "Ml", 1903, 1908, 4),
    (45, "1 Macabeus", "1Mc", 1909, 1967, 16),
    (46, "2 Macabeus", "2Mc", 1968, 2010, 15),
    (47, "S. Mateus", "Mt", 2024, 2101, 28),
    (48, "S. Marcos", "Mc", 2102, 2145, 16),
    (49, "S. Lucas", "Lc", 2146, 2219, 24),
    (50, "S. João", "Jo", 2220, 2273, 21),
    (51, "Actos dos Apóstolos", "At", 2274, 2335, 28),
    (52, "Romanos", "Rm", 2336, 2365, 16),
    (53, "1 Coríntios", "1Cor", 2366, 2393, 16),
    (54, "2 Coríntios", "2Cor", 2394, 2411, 13),
    (55, "Gálatas", "Gl", 2412, 2421, 6),
    (56, "Efésios", "Ef", 2422, 2431, 6),
    (57, "Filipenses", "Fp", 2432, 2439, 4),
    (58, "Colossenses", "Cl", 2440, 2446, 4),
    (59, "1 Tessalonicenses", "1Ts", 2447, 2453, 5),
    (60, "2 Tessalonicenses", "2Ts", 2454, 2457, 3),
    (61, "1 Timóteo", "1Tm", 2458, 2465, 6),
    (62, "2 Timóteo", "2Tm", 2466, 2471, 4),
    (63, "Tito", "Tt", 2472, 2474, 3),
    (64, "Filémon", "Fm", 2475, 2476, 1),
    (65, "Hebreus", "Hb", 2477, 2498, 13),
    (66, "S. Tiago", "Tg", 2499, 2506, 5),
    (67, "1 S. Pedro", "1Pd", 2507, 2514, 5),
    (68, "2 S. Pedro", "2Pd", 2515, 2520, 3),
    (69, "1 S. João", "1Jo", 2521, 2528, 5),
    (70, "2 S. João", "2Jo", 2529, 2530, 1),
    (71, "3 S. João", "3Jo", 2530, 2531, 1),
    (72, "S. Judas", "Jd", 2532, 2534, 1),
    (73, "Apocalipse", "Ap", 2535, 2568, 22),
]

# Pages 547 and 548 in pms.pdf are a duplicate scan of pages 549 and 550 (1 Reis 25:1-28)
DUP_PAGES = {547, 548}

ROMAN_VAL = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100}

def parse_roman(s):
    if re.match(r"^[UÚ]\s*N\s*I\s*C\s*O$", s.strip(), re.I) or "NICO" in s.upper():
        return 1
    s = (s.replace("Í", "I").replace("í", "I").replace("J", "I").replace("j", "I")
          .replace("1", "I").replace("0", "O").replace("!", "I").replace("l", "I")
          .replace("f", "I").replace("i", "I").upper().strip())
    s = re.sub(r"[^IVXLC]", "", s)
    if not s:
        return None
    total = 0
    prev = 0
    for ch in reversed(s):
        v = ROMAN_VAL.get(ch, 0)
        if v < prev:
            total -= v
        else:
            total += v
            prev = v
    return total

CAP_EXTRACT_RE = re.compile(
    r"^(?:(?:\S+\s+){0,2}?)?[\^.\s]*(?:[CGÇ]\s*[aáàâAÁÀÂ\\*°?]*\s*(?:[\*\\?35]+\s*)?[pP35\\.°*\\?]+|[aáàâAÁÀÂ]\s*[pP]\s*\.|[pP]\s*\.?\s+(?=[IVXLC]))\s*[.i,°*\s]*([IVXLC0-9ÍíJj!lfi\s]+?)\s*(?:[.,]?\s*[—\-–■\'\^~]+(?:\s*t\s*[—\-–■\'\^~]*)?|\s+1\b)"
)

FOOT_CAP_RE = re.compile(
    r"^\s*(?:[COoc]\s*[Aaa]\s*[Pp]\s*[,.\-]?|S\s*\.)\s*([IVXLCivxlc0-9ÍíJj!lfi\s]+?)\.?\s*$"
)
FOOT_NOTE_RE = re.compile(
    r"^\s*(\d+(?:\s*[-–,]\s*\d+|\s+e\s+segs?\.)*)\s*[.,\-]\s+(\S.*)"
)

SECTION_HDR_RE = re.compile(
    r"^\s*(?:"
    r"(?:PRIMEIRA|SEGUNDA|SECUNDA|TERCEIRA|QUARTA|QUINTA|SEXTA|SÉTIMA)\s+PARTE\b|"
    r"FIM\s+D[OAE]\s+(?:LIVRO|PROFECIA|EPÍSTOLA|EVANGELHO|SEGUNDO|PRIMEIRO)\b|"
    r"P\s*R\s*E\s*[ÂA]\s*M\s*B\s*U\s*L\s*O\s*$|"
    r"P\s*R\s*[ÓO]\s*L\s*O\s*G\s*O\s*$|"
    r"E\s*P\s*[ÍI]\s*L\s*O\s*G\s*O\s*$|"
    r"C\s*O\s*N\s*C\s*L\s*U\s*S\s*[ÃA]\s*O\s*$|"
    r"CANTO\s+[IVX]+\b|"
    r"[IVX]+\.?\s*[—\-–]\s+\S|"
    r"§\s*\d+"
    r")",
    re.I
)

ENCLITICS = {
    "se", "me", "te", "lhe", "lhes", "nos", "vos",
    "o", "a", "os", "as", "lo", "la", "los", "las",
    "no", "na", "nas", "lho", "lha", "lhos", "lhas",
    "mo", "ma", "mos", "mas", "to", "ta", "tos", "tas",
}

SINGLE_LETTER_WORDS = {"a", "e", "o", "à", "á", "é", "ó", "ò", "ú", "A", "E", "O", "À", "Á", "É", "Ó"}

def fix_spaced_letters(text):
    """Collapse OCR spaced-out small-caps/italics like 'O E s p irito de D e u s' or 'R e c e b a'."""
    tokens = text.split()
    if len(tokens) < 2:
        return text
    # Detect if there are runs of single-letter tokens
    out = []
    i = 0
    n = len(tokens)
    while i < n:
        # Check if we are starting a spaced-out sequence
        if len(tokens[i]) == 1 and tokens[i].isalpha() and i + 1 < n and len(tokens[i+1]) <= 2 and tokens[i+1].rstrip(".,;:!?").isalpha():
            # Collect contiguous short fragments if at least 2 single-letter tokens
            j = i
            single_cnt = 0
            while j < n:
                t_clean = tokens[j].strip(".,;:!?()")
                if len(t_clean) == 1 and t_clean.isalpha():
                    single_cnt += 1
                    j += 1
                elif len(t_clean) == 2 and t_clean.isalpha() and single_cnt >= 2 and t_clean.lower() not in {"de", "do", "da", "em", "no", "na", "os", "as", "um", "ao", "se", "me", "te", "tu", "eu", "já", "só", "fé"}:
                    j += 1
                else:
                    break
            if single_cnt >= 2 and (j - i) >= 3:
                # Reconstruct words from tokens[i:j], keeping standalone 'e', 'a', 'o' when appropriate
                buf = []
                for k in range(i, j):
                    tk = tokens[k]
                    buf.append(tk)
                    if tk[-1] in ".,;:!?)":
                        out.append("".join(buf))
                        buf = []
                if buf:
                    out.append("".join(buf))
                i = j
                continue
        out.append(tokens[i])
        i += 1
    return " ".join(out)

def join_line_words(words, h_thresh=7.0, mark_verse_gaps=False):
    if not words:
        return ""
    out = [words[0][4]]
    for prev, curr in zip(words, words[1:]):
        gap = curr[0] - prev[2]
        t1, t2 = prev[4], curr[4]
        h1, h2 = (prev[3] - prev[1]), (curr[3] - curr[1])
        c1, c2 = t1.strip(".,;:!?()"), t2.strip(".,;:!?()")
        if t1.isdigit() and t2.isdigit() and gap <= 4.0:
            out[-1] = out[-1] + t2
        elif (
            not t1.isdigit() and not t2.isdigit()
            and not t1.endswith((".", ",", ";", ":", "!", "?"))
            and (
                (max(h1, h2) < h_thresh - 0.2 and gap <= 1.75 and (len(c1) <= 2 or len(c2) <= 2))
                or (gap <= 1.15 and (len(c1) == 1 or len(c2) == 1) and c1.lower() not in {"e", "a", "o", "á", "à", "é"})
            )
        ):
            out[-1] = out[-1] + t2
        else:
            if (
                mark_verse_gaps
                and gap >= 8.2
                and t1.endswith((".", ":", ";", "!", "?", ")"))
                and not out[-1].lower().endswith(("cap.", "ap.", "p."))
                and t2 and (t2[0].isupper() or t2[0] in "«\"(")
            ):
                out.append("@@GAP@@")
            out.append(t2)
    return " ".join(out)

def join_lines_text(lines):
    """Join multiple lines of text, de-hyphenating line-break hyphens cleanly."""
    if not lines:
        return ""
    res = ""
    for line in lines:
        # Strip inline footnote reference markers at end of line (e.g. "* 1", "* 1 6", "I * 8 9", "*")
        line = re.sub(r"\s+(?:[I1l]\s+)?\*\s*(?:\d+(?:\s+\d+)*)?\s*$", "", line)
        line = re.sub(r"\*+", "", line)
        line = line.strip()
        if not line:
            continue
        if not res:
            res = line
            continue
        if res.endswith("\xad"):
            first_next = line.split()[0].rstrip(".,;:!?()") if line.split() else ""
            if first_next.lower() in ENCLITICS or "-" in first_next:
                res = res[:-1] + "-" + line
            else:
                res = res[:-1] + line
        elif res.endswith("-") and not res.endswith(" -") and not res.endswith("—"):
            first_next = line.split()[0].rstrip(".,;:!?()") if line.split() else ""
            if first_next.lower() in ENCLITICS or "-" in first_next or (first_next and first_next[0].isupper()):
                res = res + line
            else:
                res = res[:-1] + line
        else:
            res = res + " " + line
    # Also clean any internal soft hyphens followed by space
    res = re.sub(r"\xad\s+", "", res)
    res = res.replace("\xad", "")
    # Clean stray spaces before punctuation
    res = re.sub(r"\s+([.,;:!?])", r"\1", res)
    res = re.sub(r"\s+", " ", res).strip()
    return res

def load_pages():
    if not os.path.exists(BBOX_CACHE):
        print(f"Generating {BBOX_CACHE} from {PDF_PATH}...")
        subprocess.run(["pdftotext", "-bbox", PDF_PATH, BBOX_CACHE], check=True)
    pages = []
    context = etree.iterparse(BBOX_CACHE, events=("end",), tag="{http://www.w3.org/1999/xhtml}page", html=False)
    for _, elem in context:
        words = [
            (float(w.attrib["xMin"]), float(w.attrib["yMin"]), float(w.attrib["xMax"]), float(w.attrib["yMax"]), w.text or "")
            for w in elem
        ]
        pages.append(words)
        elem.clear()
    return pages

def get_page_lines(pages, pno):
    h_thresh = 6.8 if pno < 390 else (6.6 if pno < 1015 else (6.8 if pno < 2012 else 7.2))
    words = [w for w in pages[pno-1] if w[1] >= 40.0]
    main_w = [w for w in words if (w[3]-w[1]) >= h_thresh]
    if not main_w:
        return [], []
    xs0 = sorted(w[0] for w in main_w)
    xs1 = sorted(w[2] for w in main_w)
    col_x0 = xs0[len(xs0)//10]
    col_x1 = xs1[len(xs1)*9//10]
    is_even = (pno % 2 == 0) if (1019 <= pno <= 2010) else (pno % 2 == 1)
    col_w = [w for w in words if (w[3]-w[1]) >= 5.0 and w[1] >= 44.0]
    if is_even:
        valid_x0 = sorted(w[0] for w in col_w if w[0] > 48.0)
        valid_x1 = sorted(w[2] for w in col_w)
        raw_left = valid_x0[len(valid_x0)//30] if valid_x0 else 56.0
        raw_right = valid_x1[len(valid_x1)*29//30] if valid_x1 else 284.0
        page_shift = max(0.0, raw_right - 284.0)
        main_left = min(58.5 + page_shift, raw_left)
    else:
        valid_x1 = [w[2] for w in col_w if w[0] < 248.0 and w[2] < 263.0]
        raw_right = max(valid_x1) if valid_x1 else 254.0
        main_right = max(250.0, raw_right)
    filtered = []
    for w in words:
        x0, y0, x1, y1, txt = w
        h = y1 - y0
        if is_even:
            if h >= h_thresh - 0.3 and txt.strip(".,").isdigit() and x0 >= main_left - 12.5:
                filtered.append(w)
                continue
            if x1 <= main_left - 0.5 or x0 < main_left - 3.5:
                continue
        else:
            if x0 >= main_right + 0.5 or x1 > main_right + 3.5:
                continue
        filtered.append(w)
    words_sorted = sorted(filtered, key=lambda w: ((w[1]+w[3])/2.0, w[0]))
    lines = []
    for w in words_sorted:
        yc = (w[1]+w[3])/2.0
        if lines and abs(yc - lines[-1]["yc"]) <= 3.0:
            lines[-1]["words"].append(w)
            lines[-1]["yc"] = sum((x[1]+x[3])/2.0 for x in lines[-1]["words"]) / len(lines[-1]["words"])
        else:
            lines.append({"yc": yc, "words": [w]})
    for l in lines:
        l["words"].sort(key=lambda w: w[0])
        hs = sorted(w[3]-w[1] for w in l["words"])
        l["mh"] = hs[len(hs)//2]
        l["x0"] = l["words"][0][0]
        l["x1"] = l["words"][-1][2]
        l["text"] = join_line_words(l["words"], h_thresh=h_thresh, mark_verse_gaps=True)
    if lines and lines[0]["yc"] < 43.5 and re.search(r"[—,]|\b\d+\s*$|^\s*\d+\b", lines[0]["text"]):
        if not re.search(r"^\s*\d+\s+[A-ZÁÉÍÓÚÂÊÔÃÕÇ]", lines[0]["text"]):
            lines = lines[1:]
    last_main_idx = -1
    last_cap_idx = max(
        (i for i, l in enumerate(lines) if CAP_EXTRACT_RE.search(l["text"]) or "único" in l["text"] or "ú n" in l["text"] or re.search(r"^\s*S\s*A\s*L\s*M\s*O\b", l["text"], re.I)),
        default=-1
    )
    for i, l in enumerate(lines):
        dy = (l["yc"] - lines[i-1]["yc"]) if i > 0 else 0.0
        if i > last_cap_idx and last_main_idx >= 0 and l["yc"] > 220.0:
            if FOOT_CAP_RE.match(l["text"]):
                break
            if dy >= 10.0 and FOOT_NOTE_RE.match(l["text"]):
                break
            if i + 1 < len(lines) and (lines[i+1]["yc"] - l["yc"]) < 6.9 and FOOT_NOTE_RE.match(l["text"]):
                break
        if l["mh"] >= h_thresh and len(l["words"]) >= 2:
            last_main_idx = i
        elif l["mh"] >= h_thresh and len(l["words"]) == 1 and not l["text"].isdigit():
            last_main_idx = i
        elif CAP_EXTRACT_RE.search(l["text"]):
            last_main_idx = i
    return lines[:last_main_idx+1], lines[last_main_idx+1:]

def is_section_header(ml, i):
    l = ml[i]
    txt = l["text"].strip()
    if CAP_EXTRACT_RE.match(txt) or "único" in txt.lower() or "ú n" in txt.lower():
        return False
    if re.match(r"^\s*S\s*A\s*L\s*M\s*O\b", txt, re.I):
        return False
    if re.match(r"^\s*\d+\s+[A-ZÁÉÍÓÚÂÊÔÃÕÇ«\"(a-z]", txt):
        return False
    if SECTION_HDR_RE.match(txt):
        return True
    dy_prev = (l["yc"] - ml[i-1]["yc"]) if i > 0 else 20.0
    dy_next = (ml[i+1]["yc"] - l["yc"]) if i+1 < len(ml) else 20.0
    if i > 0 and dy_prev >= 18.0 and dy_next >= 14.0:
        return True
    return False

def parse_ocr_vnum(tok, cur_v):
    t = tok.strip(".,;:()")
    if t.isdigit():
        v = int(t)
        if cur_v + 1 <= v <= cur_v + 3:
            return v
        return None
    t2 = (t.replace("l", "1").replace("I", "1").replace("!", "1").replace("|", "1")
           .replace("O", "0").replace("o", "0").replace("õ", "5").replace("S", "5"))
    if t2.isdigit() and int(t2) == cur_v + 1:
        return cur_v + 1
    return None

def split_chapter_verses(bnum, cnum, clines):
    """Split a chapter's lines into [(verse_num, verse_text), ...]."""
    if bnum == 21:
        # In Salmos, strip the italic editorial summary lines before verse 1
        start_idx = 0
        for idx, line in enumerate(clines):
            if re.search(r"^\s*1\b|\b1\s+[A-ZÁÉÍÓÚÂÊÔÃÕÇ«\"(a-z]", line):
                start_idx = idx
                break
            if cnum in (115, 147) and re.search(r"^\s*1[02]\b", line):
                start_idx = idx
                break
            if cnum == 42 and re.search(r"Julga-me", line):
                start_idx = idx
                break
            if cnum == 48 and re.search(r"Para o fim", line):
                start_idx = idx
                break
        clines = clines[start_idx:]
        if clines:
            clines[0] = re.sub(r"^.*?\b1\s+", "", clines[0], count=1) if re.search(r"\b1\s+", clines[0]) else clines[0]

    joined = join_lines_text(clines)
    if not joined:
        return [(1, "")]

    tokens = joined.split()
    verses = []
    cur_v = 1
    cur_words = []

    if bnum == 21 and cnum in (115, 147):
        first_num = 10 if cnum == 115 else 12
        if tokens and tokens[0].strip(".,") == str(first_num):
            tokens = tokens[1:]
        cur_v = first_num

    for idx, tok in enumerate(tokens):
        v = parse_ocr_vnum(tok, cur_v)
        # Ensure tok isn't part of a parenthetical reference or followed by nothing
        if v is not None and idx + 1 < len(tokens) and cur_words:
            if v == cur_v + 2 and "@@GAP@@" in cur_words:
                gap_pos = cur_words.index("@@GAP@@")
                w1 = [w for w in cur_words[:gap_pos] if w != "@@GAP@@"]
                w2 = [w for w in cur_words[gap_pos+1:] if w != "@@GAP@@"]
                if w1 and w2:
                    v_out1 = (cur_v - 9) if (bnum == 21 and cnum == 115 and cur_v >= 10) else (
                             (cur_v - 11) if (bnum == 21 and cnum == 147 and cur_v >= 12) else cur_v)
                    verses.append((v_out1, " ".join(w1).strip()))
                    cur_v += 1
                    cur_words = w2
            clean_words = [w for w in cur_words if w != "@@GAP@@"]
            if clean_words:
                v_out = (cur_v - 9) if (bnum == 21 and cnum == 115 and cur_v >= 10) else (
                        (cur_v - 11) if (bnum == 21 and cnum == 147 and cur_v >= 12) else cur_v)
                verses.append((v_out, " ".join(clean_words).strip()))
            cur_v = v
            cur_words = []
        else:
            cur_words.append(tok)

    clean_words = [w for w in cur_words if w != "@@GAP@@"]
    if clean_words or not verses:
        v_out = (cur_v - 9) if (bnum == 21 and cnum == 115 and cur_v >= 10) else (
                (cur_v - 11) if (bnum == 21 and cnum == 147 and cur_v >= 12) else cur_v)
        verses.append((v_out, " ".join(clean_words).strip()))

    # Fill any skipped verse numbers if v jumped by 2 so verse numbers are strictly contiguous
    return verses

def parse_book_footnotes(bnum, foot_lines_by_page, page_first_chap):
    """Parse footnotes across a book's pages into [(chap, v_start, v_end, label, text), ...]."""
    notes = []
    cur_chap = 1
    cur_note = None  # [chap, v_start, v_end, label, lines_list]

    for pno, flines in foot_lines_by_page:
        if pno in page_first_chap and cur_chap < page_first_chap[pno] - 1:
            cur_chap = page_first_chap[pno]
        for l in flines:
            txt = l["text"].replace("@@GAP@@", "").strip()
            if not txt:
                continue
            m_cap = FOOT_CAP_RE.match(txt)
            if m_cap:
                r = parse_roman(m_cap.group(1))
                if r is not None:
                    if cur_note:
                        notes.append((cur_note[0], cur_note[1], cur_note[2], cur_note[3], join_lines_text(cur_note[4])))
                        cur_note = None
                    cur_chap = r
                continue
            m_note = FOOT_NOTE_RE.match(txt)
            if m_note:
                if cur_note:
                    notes.append((cur_note[0], cur_note[1], cur_note[2], cur_note[3], join_lines_text(cur_note[4])))
                label = m_note.group(1).strip()
                rest = m_note.group(2).strip()
                nums = [int(x) for x in re.findall(r"\d+", label)]
                v_start = nums[0] if nums else 1
                v_end = nums[-1] if len(nums) > 1 else v_start
                if v_end < v_start:
                    v_end = v_start
                cur_note = [cur_chap, v_start, v_end, label, [rest]]
            else:
                if cur_note:
                    cur_note[4].append(txt)

    if cur_note:
        notes.append((cur_note[0], cur_note[1], cur_note[2], cur_note[3], join_lines_text(cur_note[4])))
    return [n for n in notes if n[4]]

def main():
    os.makedirs(os.path.dirname(TSV_PATH), exist_ok=True)
    pages = load_pages()

    all_verses = []
    all_notes = []

    for bnum, bname, babbr, p1, p2, exp_ch in BOOKS:
        book_lines = []
        foot_lines_by_page = []
        page_first_chap = {}

        for pno in range(p1, p2+1):
            if pno in DUP_PAGES:
                continue
            ml, fl = get_page_lines(pages, pno)
            if bnum == 70 and pno == 2530:
                ml = [l for l in ml if l["yc"] < 150.0]
                fl = []
            elif bnum == 71 and pno == 2530:
                ml = [l for l in ml if l["yc"] >= 150.0]
            for i, l in enumerate(ml):
                if is_section_header(ml, i):
                    continue
                book_lines.append((pno, l["yc"], l["mh"], l["text"]))
            foot_lines_by_page.append((pno, fl))

        chapters = []
        for pno, yc, mh, txt in book_lines:
            is_ch = False
            if bnum == 21:
                m = re.search(r"^\s*S\s*A\s*L\s*M\s*O\s+([IVXLC0-9ÍíJjL!lfi\s]+)\.?\s*$", txt, re.I)
                if m:
                    cnum = len(chapters) + 1
                    chapters.append((cnum, []))
                    page_first_chap.setdefault(pno, cnum)
                    is_ch = True
            elif exp_ch == 1:
                m = re.match(
                    r"^(?:(?:\S+\s+){0,2}?)?[\^.\s]*[CGÇ]\s*[aáàâAÁÀÂ]*\s*[pP]\s*\.?\s*[uúUÚ]\s*n\s*i?\s*c\s*o\s*\.?\s*[—\-–]+\s*(?:1\b\s*)?(.*)",
                    txt, re.I
                )
                if m and not chapters:
                    chapters.append((1, [m.group(1)]))
                    page_first_chap.setdefault(pno, 1)
                    is_ch = True
            else:
                m = CAP_EXTRACT_RE.match(txt)
                if m:
                    rest = txt[m.end():].strip()
                    rest = re.sub(r"^(?:ALEF\.\s*)?1\b\s*", "", rest, flags=re.I)
                    cnum = len(chapters) + 1
                    chapters.append((cnum, [rest]))
                    page_first_chap.setdefault(pno, cnum)
                    is_ch = True
                elif re.match(r"^\s*CAP\.\s+1\s+(.*)", txt):
                    m2 = re.match(r"^\s*CAP\.\s+1\s+(.*)", txt)
                    cnum = len(chapters) + 1
                    chapters.append((cnum, [m2.group(1)]))
                    page_first_chap.setdefault(pno, cnum)
                    is_ch = True
                else:
                    m2 = re.match(r"^\s*\.?\s*([IVXLC]+)\s*[—\-–■\^~]+\s*1\b\s*(.*)", txt)
                    if m2:
                        cnum = len(chapters) + 1
                        chapters.append((cnum, [m2.group(2)]))
                        page_first_chap.setdefault(pno, cnum)
                        is_ch = True
            if not is_ch and chapters:
                chapters[-1][1].append(txt)

        if len(chapters) != exp_ch:
            print(f"WARNING: Book {bnum} ({bname}) chapters {len(chapters)} != {exp_ch}", file=sys.stderr)

        for cnum, clines in chapters:
            vlist = split_chapter_verses(bnum, cnum, clines)
            for vnum, vtext in vlist:
                vtext_clean = vtext.replace("\t", " ").strip()
                if vtext_clean:
                    all_verses.append((bname, babbr, bnum, cnum, vnum, vtext_clean))

        bnotes = parse_book_footnotes(bnum, foot_lines_by_page, page_first_chap)
        for cnum, vs, ve, label, ntext in bnotes:
            ntext_clean = ntext.replace("\t", " ").strip()
            if ntext_clean:
                all_notes.append((bnum, cnum, vs, ve, label, ntext_clean))

    # Write data/pms.tsv
    with open(TSV_PATH, "w", encoding="utf-8") as f:
        for bname, babbr, bnum, cnum, vnum, vtext in all_verses:
            f.write(f"{bname}\t{babbr}\t{bnum}\t{cnum}\t{vnum}\t{vtext}\n")

    # Write data/pms_notes.tsv
    with open(NOTES_TSV_PATH, "w", encoding="utf-8") as f:
        for bnum, cnum, vs, ve, label, ntext in all_notes:
            f.write(f"{bnum}\t{cnum}\t{vs}\t{ve}\t{label}\t{ntext}\n")

    # Index notes by (bnum, cnum) for pms.txt
    notes_by_bc = {}
    for bnum, cnum, vs, ve, label, ntext in all_notes:
        notes_by_bc.setdefault((bnum, cnum), []).append((label, ntext))

    # Write clean, human-readable pms.txt
    with open(TXT_PATH, "w", encoding="utf-8") as f:
        f.write("BÍBLIA SAGRADA\n")
        f.write("Tradução e Comentários do Padre Manuel de Matos Soares (1932)\n")
        f.write("=============================================================\n\n")
        cur_b = None
        cur_c = None
        for bname, babbr, bnum, cnum, vnum, vtext in all_verses:
            if bnum != cur_b:
                if cur_b is not None and (cur_b, cur_c) in notes_by_bc:
                    f.write("\n  [Notas — Capítulo " + str(cur_c) + "]\n")
                    for lbl, ntxt in notes_by_bc[(cur_b, cur_c)]:
                        f.write(f"  * {lbl}. {ntxt}\n")
                cur_b = bnum
                cur_c = None
                f.write(f"\n\n=============================================================\n")
                f.write(f"LIVRO {bnum}: {bname.upper()} ({babbr})\n")
                f.write(f"=============================================================\n")
            if cnum != cur_c:
                if cur_c is not None and (bnum, cur_c) in notes_by_bc:
                    f.write("\n  [Notas — Capítulo " + str(cur_c) + "]\n")
                    for lbl, ntxt in notes_by_bc[(bnum, cur_c)]:
                        f.write(f"  * {lbl}. {ntxt}\n")
                cur_c = cnum
                hdr_label = "Salmo" if bnum == 21 else "Capítulo"
                f.write(f"\n--- {bname} — {hdr_label} {cnum} ---\n\n")
            f.write(f"{cnum}:{vnum} {vtext}\n")
        if cur_b is not None and (cur_b, cur_c) in notes_by_bc:
            f.write("\n  [Notas — Capítulo " + str(cur_c) + "]\n")
            for lbl, ntxt in notes_by_bc[(cur_b, cur_c)]:
                f.write(f"  * {lbl}. {ntxt}\n")

    print(f"Successfully extracted {len(all_verses)} verses and {len(all_notes)} commentary notes across {len(BOOKS)} books.")

if __name__ == "__main__":
    main()