diff options
Diffstat (limited to 'scripts/extract_pms.py')
| -rwxr-xr-x | scripts/extract_pms.py | 640 |
1 files changed, 640 insertions, 0 deletions
diff --git a/scripts/extract_pms.py b/scripts/extract_pms.py new file mode 100755 index 0000000..22aa1e3 --- /dev/null +++ b/scripts/extract_pms.py @@ -0,0 +1,640 @@ +#!/usr/bin/env python3 +""" +Extract the Padre Matos Soares (1932) Catholic Bible from pms.pdf into: + - data/pms.tsv (BookName, BookAbbr, BookNum, Chapter, Verse, Text) + - data/pms_notes.tsv (BookNum, Chapter, VerseStart, VerseEnd, Label, NoteText) + - pms.txt (Clean, human-readable formatted text of the Bible + notes) +""" + +import os +import re +import subprocess +import sys +from lxml import etree + +REPO_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PDF_PATH = os.path.join(REPO_DIR, "pms.pdf") +BBOX_CACHE = "/tmp/pms_bbox.html" +TSV_PATH = os.path.join(REPO_DIR, "data", "pms.tsv") +NOTES_TSV_PATH = os.path.join(REPO_DIR, "data", "pms_notes.tsv") +TXT_PATH = os.path.join(REPO_DIR, "pms.txt") + +BOOKS = [ + (1, "Génesis", "Gn", 18, 115, 50), + (2, "Êxodo", "Ex", 116, 193, 40), + (3, "Levítico", "Lv", 194, 245, 27), + (4, "Números", "Nm", 246, 319, 36), + (5, "Deuteronómio", "Dt", 320, 387, 34), + (6, "Josué", "Js", 398, 441, 24), + (7, "Juízes", "Jz", 442, 489, 21), + (8, "Rut", "Rt", 490, 497, 4), + (9, "1 Reis", "1Rs", 498, 561, 31), + (10, "2 Reis", "2Rs", 562, 611, 24), + (11, "3 Reis", "3Rs", 612, 667, 22), + (12, "4 Reis", "4Rs", 668, 719, 25), + (13, "1 Paralipómenos", "1Par", 720, 769, 29), + (14, "2 Paralipómenos", "2Par", 770, 831, 36), + (15, "1 Esdras", "1Esd", 832, 850, 10), + (16, "2 Esdras (Neemias)", "2Esd", 851, 876, 13), + (17, "Tobias", "Tb", 877, 894, 14), + (18, "Judit", "Jt", 895, 918, 16), + (19, "Ester", "Est", 919, 941, 16), + (20, "Job", "Job", 942, 1013, 42), + (21, "Salmos", "Sl", 1019, 1196, 150), + (22, "Provérbios", "Pr", 1197, 1255, 31), + (23, "Eclesiastes", "Ecl", 1256, 1272, 12), + (24, "Cântico dos Cânticos", "Ct", 1273, 1287, 8), + (25, "Sabedoria", "Sb", 1288, 1329, 19), + (26, "Eclesiástico", "Eclo", 1330, 1439, 51), + (27, "Isaías", "Is", 1440, 1543, 66), + (28, "Jeremias", "Jr", 1544, 1647, 52), + (29, "Lamentações", "Lm", 1648, 1661, 5), + (30, "Baruc", "Br", 1662, 1675, 6), + (31, "Ezequiel", "Ez", 1676, 1774, 48), + (32, "Daniel", "Dn", 1775, 1816, 14), + (33, "Oséas", "Os", 1817, 1833, 14), + (34, "Joel", "Jl", 1834, 1839, 3), + (35, "Amós", "Am", 1840, 1851, 9), + (36, "Abdias", "Abd", 1852, 1853, 1), + (37, "Jonas", "Jn", 1854, 1857, 4), + (38, "Miquéas", "Mq", 1858, 1866, 7), + (39, "Naum", "Na", 1867, 1870, 3), + (40, "Habacuc", "Hab", 1871, 1875, 3), + (41, "Sofonias", "Sf", 1876, 1880, 3), + (42, "Ageu", "Ag", 1881, 1884, 2), + (43, "Zacarias", "Zc", 1885, 1902, 14), + (44, "Malaquias", "Ml", 1903, 1908, 4), + (45, "1 Macabeus", "1Mc", 1909, 1967, 16), + (46, "2 Macabeus", "2Mc", 1968, 2010, 15), + (47, "S. Mateus", "Mt", 2024, 2101, 28), + (48, "S. Marcos", "Mc", 2102, 2145, 16), + (49, "S. Lucas", "Lc", 2146, 2219, 24), + (50, "S. João", "Jo", 2220, 2273, 21), + (51, "Actos dos Apóstolos", "At", 2274, 2335, 28), + (52, "Romanos", "Rm", 2336, 2365, 16), + (53, "1 Coríntios", "1Cor", 2366, 2393, 16), + (54, "2 Coríntios", "2Cor", 2394, 2411, 13), + (55, "Gálatas", "Gl", 2412, 2421, 6), + (56, "Efésios", "Ef", 2422, 2431, 6), + (57, "Filipenses", "Fp", 2432, 2439, 4), + (58, "Colossenses", "Cl", 2440, 2446, 4), + (59, "1 Tessalonicenses", "1Ts", 2447, 2453, 5), + (60, "2 Tessalonicenses", "2Ts", 2454, 2457, 3), + (61, "1 Timóteo", "1Tm", 2458, 2465, 6), + (62, "2 Timóteo", "2Tm", 2466, 2471, 4), + (63, "Tito", "Tt", 2472, 2474, 3), + (64, "Filémon", "Fm", 2475, 2476, 1), + (65, "Hebreus", "Hb", 2477, 2498, 13), + (66, "S. Tiago", "Tg", 2499, 2506, 5), + (67, "1 S. Pedro", "1Pd", 2507, 2514, 5), + (68, "2 S. Pedro", "2Pd", 2515, 2520, 3), + (69, "1 S. João", "1Jo", 2521, 2528, 5), + (70, "2 S. João", "2Jo", 2529, 2530, 1), + (71, "3 S. João", "3Jo", 2530, 2531, 1), + (72, "S. Judas", "Jd", 2532, 2534, 1), + (73, "Apocalipse", "Ap", 2535, 2568, 22), +] + +# Pages 547 and 548 in pms.pdf are a duplicate scan of pages 549 and 550 (1 Reis 25:1-28) +DUP_PAGES = {547, 548} + +ROMAN_VAL = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100} + +def parse_roman(s): + if re.match(r"^[UÚ]\s*N\s*I\s*C\s*O$", s.strip(), re.I) or "NICO" in s.upper(): + return 1 + s = (s.replace("Í", "I").replace("í", "I").replace("J", "I").replace("j", "I") + .replace("1", "I").replace("0", "O").replace("!", "I").replace("l", "I") + .replace("f", "I").replace("i", "I").upper().strip()) + s = re.sub(r"[^IVXLC]", "", s) + if not s: + return None + total = 0 + prev = 0 + for ch in reversed(s): + v = ROMAN_VAL.get(ch, 0) + if v < prev: + total -= v + else: + total += v + prev = v + return total + +CAP_EXTRACT_RE = re.compile( + r"^(?:(?:\S+\s+){0,2}?)?[\^.\s]*(?:[CGÇ]\s*[aáàâAÁÀÂ\\*°?]*\s*(?:[\*\\?35]+\s*)?[pP35\\.°*\\?]+|[aáàâAÁÀÂ]\s*[pP]\s*\.|[pP]\s*\.?\s+(?=[IVXLC]))\s*[.i,°*\s]*([IVXLC0-9ÍíJj!lfi\s]+?)\s*(?:[.,]?\s*[—\-–■\'\^~]+(?:\s*t\s*[—\-–■\'\^~]*)?|\s+1\b)" +) + +FOOT_CAP_RE = re.compile( + r"^\s*(?:[COoc]\s*[Aaa]\s*[Pp]\s*[,.\-]?|S\s*\.)\s*([IVXLCivxlc0-9ÍíJj!lfi\s]+?)\.?\s*$" +) +FOOT_NOTE_RE = re.compile( + r"^\s*(\d+(?:\s*[-–,]\s*\d+|\s+e\s+segs?\.)*)\s*[.,\-]\s+(\S.*)" +) + +SECTION_HDR_RE = re.compile( + r"^\s*(?:" + r"(?:PRIMEIRA|SEGUNDA|SECUNDA|TERCEIRA|QUARTA|QUINTA|SEXTA|SÉTIMA)\s+PARTE\b|" + r"FIM\s+D[OAE]\s+(?:LIVRO|PROFECIA|EPÍSTOLA|EVANGELHO|SEGUNDO|PRIMEIRO)\b|" + r"P\s*R\s*E\s*[ÂA]\s*M\s*B\s*U\s*L\s*O\s*$|" + r"P\s*R\s*[ÓO]\s*L\s*O\s*G\s*O\s*$|" + r"E\s*P\s*[ÍI]\s*L\s*O\s*G\s*O\s*$|" + r"C\s*O\s*N\s*C\s*L\s*U\s*S\s*[ÃA]\s*O\s*$|" + r"CANTO\s+[IVX]+\b|" + r"[IVX]+\.?\s*[—\-–]\s+\S|" + r"§\s*\d+" + r")", + re.I +) + +ENCLITICS = { + "se", "me", "te", "lhe", "lhes", "nos", "vos", + "o", "a", "os", "as", "lo", "la", "los", "las", + "no", "na", "nas", "lho", "lha", "lhos", "lhas", + "mo", "ma", "mos", "mas", "to", "ta", "tos", "tas", +} + +SINGLE_LETTER_WORDS = {"a", "e", "o", "à", "á", "é", "ó", "ò", "ú", "A", "E", "O", "À", "Á", "É", "Ó"} + +def fix_spaced_letters(text): + """Collapse OCR spaced-out small-caps/italics like 'O E s p irito de D e u s' or 'R e c e b a'.""" + tokens = text.split() + if len(tokens) < 2: + return text + # Detect if there are runs of single-letter tokens + out = [] + i = 0 + n = len(tokens) + while i < n: + # Check if we are starting a spaced-out sequence + if len(tokens[i]) == 1 and tokens[i].isalpha() and i + 1 < n and len(tokens[i+1]) <= 2 and tokens[i+1].rstrip(".,;:!?").isalpha(): + # Collect contiguous short fragments if at least 2 single-letter tokens + j = i + single_cnt = 0 + while j < n: + t_clean = tokens[j].strip(".,;:!?()") + if len(t_clean) == 1 and t_clean.isalpha(): + single_cnt += 1 + j += 1 + elif len(t_clean) == 2 and t_clean.isalpha() and single_cnt >= 2 and t_clean.lower() not in {"de", "do", "da", "em", "no", "na", "os", "as", "um", "ao", "se", "me", "te", "tu", "eu", "já", "só", "fé"}: + j += 1 + else: + break + if single_cnt >= 2 and (j - i) >= 3: + # Reconstruct words from tokens[i:j], keeping standalone 'e', 'a', 'o' when appropriate + buf = [] + for k in range(i, j): + tk = tokens[k] + buf.append(tk) + if tk[-1] in ".,;:!?)": + out.append("".join(buf)) + buf = [] + if buf: + out.append("".join(buf)) + i = j + continue + out.append(tokens[i]) + i += 1 + return " ".join(out) + +def join_line_words(words, h_thresh=7.0, mark_verse_gaps=False): + if not words: + return "" + out = [words[0][4]] + for prev, curr in zip(words, words[1:]): + gap = curr[0] - prev[2] + t1, t2 = prev[4], curr[4] + h1, h2 = (prev[3] - prev[1]), (curr[3] - curr[1]) + c1, c2 = t1.strip(".,;:!?()"), t2.strip(".,;:!?()") + if t1.isdigit() and t2.isdigit() and gap <= 4.0: + out[-1] = out[-1] + t2 + elif ( + not t1.isdigit() and not t2.isdigit() + and not t1.endswith((".", ",", ";", ":", "!", "?")) + and ( + (max(h1, h2) < h_thresh - 0.2 and gap <= 1.75 and (len(c1) <= 2 or len(c2) <= 2)) + or (gap <= 1.15 and (len(c1) == 1 or len(c2) == 1) and c1.lower() not in {"e", "a", "o", "á", "à", "é"}) + ) + ): + out[-1] = out[-1] + t2 + else: + if ( + mark_verse_gaps + and gap >= 8.2 + and t1.endswith((".", ":", ";", "!", "?", ")")) + and not out[-1].lower().endswith(("cap.", "ap.", "p.")) + and t2 and (t2[0].isupper() or t2[0] in "«\"(") + ): + out.append("@@GAP@@") + out.append(t2) + return " ".join(out) + +def join_lines_text(lines): + """Join multiple lines of text, de-hyphenating line-break hyphens cleanly.""" + if not lines: + return "" + res = "" + for line in lines: + # Strip inline footnote reference markers at end of line (e.g. "* 1", "* 1 6", "I * 8 9", "*") + line = re.sub(r"\s+(?:[I1l]\s+)?\*\s*(?:\d+(?:\s+\d+)*)?\s*$", "", line) + line = re.sub(r"\*+", "", line) + line = line.strip() + if not line: + continue + if not res: + res = line + continue + if res.endswith("\xad"): + first_next = line.split()[0].rstrip(".,;:!?()") if line.split() else "" + if first_next.lower() in ENCLITICS or "-" in first_next: + res = res[:-1] + "-" + line + else: + res = res[:-1] + line + elif res.endswith("-") and not res.endswith(" -") and not res.endswith("—"): + first_next = line.split()[0].rstrip(".,;:!?()") if line.split() else "" + if first_next.lower() in ENCLITICS or "-" in first_next or (first_next and first_next[0].isupper()): + res = res + line + else: + res = res[:-1] + line + else: + res = res + " " + line + # Also clean any internal soft hyphens followed by space + res = re.sub(r"\xad\s+", "", res) + res = res.replace("\xad", "") + # Clean stray spaces before punctuation + res = re.sub(r"\s+([.,;:!?])", r"\1", res) + res = re.sub(r"\s+", " ", res).strip() + return res + +def load_pages(): + if not os.path.exists(BBOX_CACHE): + print(f"Generating {BBOX_CACHE} from {PDF_PATH}...") + subprocess.run(["pdftotext", "-bbox", PDF_PATH, BBOX_CACHE], check=True) + pages = [] + context = etree.iterparse(BBOX_CACHE, events=("end",), tag="{http://www.w3.org/1999/xhtml}page", html=False) + for _, elem in context: + words = [ + (float(w.attrib["xMin"]), float(w.attrib["yMin"]), float(w.attrib["xMax"]), float(w.attrib["yMax"]), w.text or "") + for w in elem + ] + pages.append(words) + elem.clear() + return pages + +def get_page_lines(pages, pno): + h_thresh = 6.8 if pno < 390 else (6.6 if pno < 1015 else (6.8 if pno < 2012 else 7.2)) + words = [w for w in pages[pno-1] if w[1] >= 40.0] + main_w = [w for w in words if (w[3]-w[1]) >= h_thresh] + if not main_w: + return [], [] + xs0 = sorted(w[0] for w in main_w) + xs1 = sorted(w[2] for w in main_w) + col_x0 = xs0[len(xs0)//10] + col_x1 = xs1[len(xs1)*9//10] + is_even = (pno % 2 == 0) if (1019 <= pno <= 2010) else (pno % 2 == 1) + col_w = [w for w in words if (w[3]-w[1]) >= 5.0 and w[1] >= 44.0] + if is_even: + valid_x0 = sorted(w[0] for w in col_w if w[0] > 48.0) + valid_x1 = sorted(w[2] for w in col_w) + raw_left = valid_x0[len(valid_x0)//30] if valid_x0 else 56.0 + raw_right = valid_x1[len(valid_x1)*29//30] if valid_x1 else 284.0 + page_shift = max(0.0, raw_right - 284.0) + main_left = min(58.5 + page_shift, raw_left) + else: + valid_x1 = [w[2] for w in col_w if w[0] < 248.0 and w[2] < 263.0] + raw_right = max(valid_x1) if valid_x1 else 254.0 + main_right = max(250.0, raw_right) + filtered = [] + for w in words: + x0, y0, x1, y1, txt = w + h = y1 - y0 + if is_even: + if h >= h_thresh - 0.3 and txt.strip(".,").isdigit() and x0 >= main_left - 12.5: + filtered.append(w) + continue + if x1 <= main_left - 0.5 or x0 < main_left - 3.5: + continue + else: + if x0 >= main_right + 0.5 or x1 > main_right + 3.5: + continue + filtered.append(w) + words_sorted = sorted(filtered, key=lambda w: ((w[1]+w[3])/2.0, w[0])) + lines = [] + for w in words_sorted: + yc = (w[1]+w[3])/2.0 + if lines and abs(yc - lines[-1]["yc"]) <= 3.0: + lines[-1]["words"].append(w) + lines[-1]["yc"] = sum((x[1]+x[3])/2.0 for x in lines[-1]["words"]) / len(lines[-1]["words"]) + else: + lines.append({"yc": yc, "words": [w]}) + for l in lines: + l["words"].sort(key=lambda w: w[0]) + hs = sorted(w[3]-w[1] for w in l["words"]) + l["mh"] = hs[len(hs)//2] + l["x0"] = l["words"][0][0] + l["x1"] = l["words"][-1][2] + l["text"] = join_line_words(l["words"], h_thresh=h_thresh, mark_verse_gaps=True) + if lines and lines[0]["yc"] < 43.5 and re.search(r"[—,]|\b\d+\s*$|^\s*\d+\b", lines[0]["text"]): + if not re.search(r"^\s*\d+\s+[A-ZÁÉÍÓÚÂÊÔÃÕÇ]", lines[0]["text"]): + lines = lines[1:] + last_main_idx = -1 + last_cap_idx = max( + (i for i, l in enumerate(lines) if CAP_EXTRACT_RE.search(l["text"]) or "único" in l["text"] or "ú n" in l["text"] or re.search(r"^\s*S\s*A\s*L\s*M\s*O\b", l["text"], re.I)), + default=-1 + ) + for i, l in enumerate(lines): + dy = (l["yc"] - lines[i-1]["yc"]) if i > 0 else 0.0 + if i > last_cap_idx and last_main_idx >= 0 and l["yc"] > 220.0: + if FOOT_CAP_RE.match(l["text"]): + break + if dy >= 10.0 and FOOT_NOTE_RE.match(l["text"]): + break + if i + 1 < len(lines) and (lines[i+1]["yc"] - l["yc"]) < 6.9 and FOOT_NOTE_RE.match(l["text"]): + break + if l["mh"] >= h_thresh and len(l["words"]) >= 2: + last_main_idx = i + elif l["mh"] >= h_thresh and len(l["words"]) == 1 and not l["text"].isdigit(): + last_main_idx = i + elif CAP_EXTRACT_RE.search(l["text"]): + last_main_idx = i + return lines[:last_main_idx+1], lines[last_main_idx+1:] + +def is_section_header(ml, i): + l = ml[i] + txt = l["text"].strip() + if CAP_EXTRACT_RE.match(txt) or "único" in txt.lower() or "ú n" in txt.lower(): + return False + if re.match(r"^\s*S\s*A\s*L\s*M\s*O\b", txt, re.I): + return False + if re.match(r"^\s*\d+\s+[A-ZÁÉÍÓÚÂÊÔÃÕÇ«\"(a-z]", txt): + return False + if SECTION_HDR_RE.match(txt): + return True + dy_prev = (l["yc"] - ml[i-1]["yc"]) if i > 0 else 20.0 + dy_next = (ml[i+1]["yc"] - l["yc"]) if i+1 < len(ml) else 20.0 + if i > 0 and dy_prev >= 18.0 and dy_next >= 14.0: + return True + return False + +def parse_ocr_vnum(tok, cur_v): + t = tok.strip(".,;:()") + if t.isdigit(): + v = int(t) + if cur_v + 1 <= v <= cur_v + 3: + return v + return None + t2 = (t.replace("l", "1").replace("I", "1").replace("!", "1").replace("|", "1") + .replace("O", "0").replace("o", "0").replace("õ", "5").replace("S", "5")) + if t2.isdigit() and int(t2) == cur_v + 1: + return cur_v + 1 + return None + +def split_chapter_verses(bnum, cnum, clines): + """Split a chapter's lines into [(verse_num, verse_text), ...].""" + if bnum == 21: + # In Salmos, strip the italic editorial summary lines before verse 1 + start_idx = 0 + for idx, line in enumerate(clines): + if re.search(r"^\s*1\b|\b1\s+[A-ZÁÉÍÓÚÂÊÔÃÕÇ«\"(a-z]", line): + start_idx = idx + break + if cnum in (115, 147) and re.search(r"^\s*1[02]\b", line): + start_idx = idx + break + if cnum == 42 and re.search(r"Julga-me", line): + start_idx = idx + break + if cnum == 48 and re.search(r"Para o fim", line): + start_idx = idx + break + clines = clines[start_idx:] + if clines: + clines[0] = re.sub(r"^.*?\b1\s+", "", clines[0], count=1) if re.search(r"\b1\s+", clines[0]) else clines[0] + + joined = join_lines_text(clines) + if not joined: + return [(1, "")] + + tokens = joined.split() + verses = [] + cur_v = 1 + cur_words = [] + + if bnum == 21 and cnum in (115, 147): + first_num = 10 if cnum == 115 else 12 + if tokens and tokens[0].strip(".,") == str(first_num): + tokens = tokens[1:] + cur_v = first_num + + for idx, tok in enumerate(tokens): + v = parse_ocr_vnum(tok, cur_v) + # Ensure tok isn't part of a parenthetical reference or followed by nothing + if v is not None and idx + 1 < len(tokens) and cur_words: + if v == cur_v + 2 and "@@GAP@@" in cur_words: + gap_pos = cur_words.index("@@GAP@@") + w1 = [w for w in cur_words[:gap_pos] if w != "@@GAP@@"] + w2 = [w for w in cur_words[gap_pos+1:] if w != "@@GAP@@"] + if w1 and w2: + v_out1 = (cur_v - 9) if (bnum == 21 and cnum == 115 and cur_v >= 10) else ( + (cur_v - 11) if (bnum == 21 and cnum == 147 and cur_v >= 12) else cur_v) + verses.append((v_out1, " ".join(w1).strip())) + cur_v += 1 + cur_words = w2 + clean_words = [w for w in cur_words if w != "@@GAP@@"] + if clean_words: + v_out = (cur_v - 9) if (bnum == 21 and cnum == 115 and cur_v >= 10) else ( + (cur_v - 11) if (bnum == 21 and cnum == 147 and cur_v >= 12) else cur_v) + verses.append((v_out, " ".join(clean_words).strip())) + cur_v = v + cur_words = [] + else: + cur_words.append(tok) + + clean_words = [w for w in cur_words if w != "@@GAP@@"] + if clean_words or not verses: + v_out = (cur_v - 9) if (bnum == 21 and cnum == 115 and cur_v >= 10) else ( + (cur_v - 11) if (bnum == 21 and cnum == 147 and cur_v >= 12) else cur_v) + verses.append((v_out, " ".join(clean_words).strip())) + + # Fill any skipped verse numbers if v jumped by 2 so verse numbers are strictly contiguous + return verses + +def parse_book_footnotes(bnum, foot_lines_by_page, page_first_chap): + """Parse footnotes across a book's pages into [(chap, v_start, v_end, label, text), ...].""" + notes = [] + cur_chap = 1 + cur_note = None # [chap, v_start, v_end, label, lines_list] + + for pno, flines in foot_lines_by_page: + if pno in page_first_chap and cur_chap < page_first_chap[pno] - 1: + cur_chap = page_first_chap[pno] + for l in flines: + txt = l["text"].replace("@@GAP@@", "").strip() + if not txt: + continue + m_cap = FOOT_CAP_RE.match(txt) + if m_cap: + r = parse_roman(m_cap.group(1)) + if r is not None: + if cur_note: + notes.append((cur_note[0], cur_note[1], cur_note[2], cur_note[3], join_lines_text(cur_note[4]))) + cur_note = None + cur_chap = r + continue + m_note = FOOT_NOTE_RE.match(txt) + if m_note: + if cur_note: + notes.append((cur_note[0], cur_note[1], cur_note[2], cur_note[3], join_lines_text(cur_note[4]))) + label = m_note.group(1).strip() + rest = m_note.group(2).strip() + nums = [int(x) for x in re.findall(r"\d+", label)] + v_start = nums[0] if nums else 1 + v_end = nums[-1] if len(nums) > 1 else v_start + if v_end < v_start: + v_end = v_start + cur_note = [cur_chap, v_start, v_end, label, [rest]] + else: + if cur_note: + cur_note[4].append(txt) + + if cur_note: + notes.append((cur_note[0], cur_note[1], cur_note[2], cur_note[3], join_lines_text(cur_note[4]))) + return [n for n in notes if n[4]] + +def main(): + os.makedirs(os.path.dirname(TSV_PATH), exist_ok=True) + pages = load_pages() + + all_verses = [] + all_notes = [] + + for bnum, bname, babbr, p1, p2, exp_ch in BOOKS: + book_lines = [] + foot_lines_by_page = [] + page_first_chap = {} + + for pno in range(p1, p2+1): + if pno in DUP_PAGES: + continue + ml, fl = get_page_lines(pages, pno) + if bnum == 70 and pno == 2530: + ml = [l for l in ml if l["yc"] < 150.0] + fl = [] + elif bnum == 71 and pno == 2530: + ml = [l for l in ml if l["yc"] >= 150.0] + for i, l in enumerate(ml): + if is_section_header(ml, i): + continue + book_lines.append((pno, l["yc"], l["mh"], l["text"])) + foot_lines_by_page.append((pno, fl)) + + chapters = [] + for pno, yc, mh, txt in book_lines: + is_ch = False + if bnum == 21: + m = re.search(r"^\s*S\s*A\s*L\s*M\s*O\s+([IVXLC0-9ÍíJjL!lfi\s]+)\.?\s*$", txt, re.I) + if m: + cnum = len(chapters) + 1 + chapters.append((cnum, [])) + page_first_chap.setdefault(pno, cnum) + is_ch = True + elif exp_ch == 1: + m = re.match( + r"^(?:(?:\S+\s+){0,2}?)?[\^.\s]*[CGÇ]\s*[aáàâAÁÀÂ]*\s*[pP]\s*\.?\s*[uúUÚ]\s*n\s*i?\s*c\s*o\s*\.?\s*[—\-–]+\s*(?:1\b\s*)?(.*)", + txt, re.I + ) + if m and not chapters: + chapters.append((1, [m.group(1)])) + page_first_chap.setdefault(pno, 1) + is_ch = True + else: + m = CAP_EXTRACT_RE.match(txt) + if m: + rest = txt[m.end():].strip() + rest = re.sub(r"^(?:ALEF\.\s*)?1\b\s*", "", rest, flags=re.I) + cnum = len(chapters) + 1 + chapters.append((cnum, [rest])) + page_first_chap.setdefault(pno, cnum) + is_ch = True + elif re.match(r"^\s*CAP\.\s+1\s+(.*)", txt): + m2 = re.match(r"^\s*CAP\.\s+1\s+(.*)", txt) + cnum = len(chapters) + 1 + chapters.append((cnum, [m2.group(1)])) + page_first_chap.setdefault(pno, cnum) + is_ch = True + else: + m2 = re.match(r"^\s*\.?\s*([IVXLC]+)\s*[—\-–■\^~]+\s*1\b\s*(.*)", txt) + if m2: + cnum = len(chapters) + 1 + chapters.append((cnum, [m2.group(2)])) + page_first_chap.setdefault(pno, cnum) + is_ch = True + if not is_ch and chapters: + chapters[-1][1].append(txt) + + if len(chapters) != exp_ch: + print(f"WARNING: Book {bnum} ({bname}) chapters {len(chapters)} != {exp_ch}", file=sys.stderr) + + for cnum, clines in chapters: + vlist = split_chapter_verses(bnum, cnum, clines) + for vnum, vtext in vlist: + vtext_clean = vtext.replace("\t", " ").strip() + if vtext_clean: + all_verses.append((bname, babbr, bnum, cnum, vnum, vtext_clean)) + + bnotes = parse_book_footnotes(bnum, foot_lines_by_page, page_first_chap) + for cnum, vs, ve, label, ntext in bnotes: + ntext_clean = ntext.replace("\t", " ").strip() + if ntext_clean: + all_notes.append((bnum, cnum, vs, ve, label, ntext_clean)) + + # Write data/pms.tsv + with open(TSV_PATH, "w", encoding="utf-8") as f: + for bname, babbr, bnum, cnum, vnum, vtext in all_verses: + f.write(f"{bname}\t{babbr}\t{bnum}\t{cnum}\t{vnum}\t{vtext}\n") + + # Write data/pms_notes.tsv + with open(NOTES_TSV_PATH, "w", encoding="utf-8") as f: + for bnum, cnum, vs, ve, label, ntext in all_notes: + f.write(f"{bnum}\t{cnum}\t{vs}\t{ve}\t{label}\t{ntext}\n") + + # Index notes by (bnum, cnum) for pms.txt + notes_by_bc = {} + for bnum, cnum, vs, ve, label, ntext in all_notes: + notes_by_bc.setdefault((bnum, cnum), []).append((label, ntext)) + + # Write clean, human-readable pms.txt + with open(TXT_PATH, "w", encoding="utf-8") as f: + f.write("BÍBLIA SAGRADA\n") + f.write("Tradução e Comentários do Padre Manuel de Matos Soares (1932)\n") + f.write("=============================================================\n\n") + cur_b = None + cur_c = None + for bname, babbr, bnum, cnum, vnum, vtext in all_verses: + if bnum != cur_b: + if cur_b is not None and (cur_b, cur_c) in notes_by_bc: + f.write("\n [Notas — Capítulo " + str(cur_c) + "]\n") + for lbl, ntxt in notes_by_bc[(cur_b, cur_c)]: + f.write(f" * {lbl}. {ntxt}\n") + cur_b = bnum + cur_c = None + f.write(f"\n\n=============================================================\n") + f.write(f"LIVRO {bnum}: {bname.upper()} ({babbr})\n") + f.write(f"=============================================================\n") + if cnum != cur_c: + if cur_c is not None and (bnum, cur_c) in notes_by_bc: + f.write("\n [Notas — Capítulo " + str(cur_c) + "]\n") + for lbl, ntxt in notes_by_bc[(bnum, cur_c)]: + f.write(f" * {lbl}. {ntxt}\n") + cur_c = cnum + hdr_label = "Salmo" if bnum == 21 else "Capítulo" + f.write(f"\n--- {bname} — {hdr_label} {cnum} ---\n\n") + f.write(f"{cnum}:{vnum} {vtext}\n") + if cur_b is not None and (cur_b, cur_c) in notes_by_bc: + f.write("\n [Notas — Capítulo " + str(cur_c) + "]\n") + for lbl, ntxt in notes_by_bc[(cur_b, cur_c)]: + f.write(f" * {lbl}. {ntxt}\n") + + print(f"Successfully extracted {len(all_verses)} verses and {len(all_notes)} commentary notes across {len(BOOKS)} books.") + +if __name__ == "__main__": + main() |