#!/usr/bin/env python3
"""Compare exact substring, spelling-tolerant and line-break-normalized
Japanese archive searches.

Run: python3 archive-search-ja.py
Requires Python 3 with SQLite FTS5 and its trigram tokenizer.
The invented Japanese corpus illustrates search behavior, not client results.
"""

import re
import sqlite3
import sys


CORPUS = {
    "A": "海運業は明治時代に始まった。海運業は石炭を運んだ。",
    "B": "毎運業は石炭を運んだ。",
    "C": "かいうんぎょうは石炭を運んだ。",
    "D": "海苔業の養殖が盛んになった。",
    "E": "海-\n運業は石炭を運んだ。",
    "F": "石炭は海運業で運ばれた。",
}
CANONICAL = "海運業"
# A small, explicit candidate list. Do not treat this as a production vocabulary.
CANDIDATES = ["毎運業", "海苔業"]
ALIASES = {"かいうんぎょう": CANONICAL}


def edit_distance(left, right):
    previous = list(range(len(right) + 1))
    for i, a in enumerate(left, 1):
        current = [i]
        for j, b in enumerate(right, 1):
            current.append(min(current[-1] + 1, previous[j] + 1,
                               previous[j - 1] + (a != b)))
        previous = current
    return previous[-1]


def join_line_end_hyphen(text):
    """Illustrative rule: join a word broken by a hyphen and newline."""
    return re.sub(r"-\r?\n", "", text)


def count(text, term):
    return text.count(term)


def main():
    db = sqlite3.connect(":memory:")
    try:
        db.execute("CREATE VIRTUAL TABLE raw USING fts5(id UNINDEXED, body, "
                   "tokenize='trigram')")
        db.execute("CREATE VIRTUAL TABLE normalized USING fts5("
                   "id UNINDEXED, body, tokenize='trigram')")
    except sqlite3.OperationalError as error:
        raise SystemExit("This example requires SQLite with FTS5 and the "
                         "trigram tokenizer: " + str(error)) from error
    db.executemany("INSERT INTO raw(id, body) VALUES (?, ?)", CORPUS.items())
    normalized = {key: join_line_end_hyphen(value)
                  for key, value in CORPUS.items()}
    db.executemany("INSERT INTO normalized(id, body) VALUES (?, ?)",
                   normalized.items())

    def documents(table, query):
        return [row[0] for row in db.execute(
            f"SELECT id FROM {table} WHERE {table} MATCH ? ORDER BY id",
            ('"' + query.replace('"', '""') + '"',))]

    def occurrences(table, term):
        return sum(count(normalized[key], term) if table == "normalized"
                   else count(CORPUS[key], term) for key in CORPUS)

    raw_docs = documents("raw", CANONICAL)
    raw_count = occurrences("raw", CANONICAL)
    normalized_docs = documents("normalized", CANONICAL)
    normalized_count = occurrences("normalized", CANONICAL)

    fuzzy_terms = []
    for term in CANDIDATES:
        if edit_distance(CANONICAL, term) <= 1:
            fuzzy_terms.append(term)
    fuzzy_terms.extend(ALIASES)
    fuzzy_docs = []
    fuzzy_count = 0
    for term in fuzzy_terms:
        matched = documents("normalized", term)
        fuzzy_docs.extend(matched)
        fuzzy_count += occurrences("normalized", term)
    fuzzy_docs = sorted(set(fuzzy_docs))
    expanded_docs = sorted(set(raw_docs) | set(normalized_docs) | set(fuzzy_docs))
    expanded_count = normalized_count + fuzzy_count

    checks = [
        (raw_docs == ["A", "F"], "raw exact documents"),
        (raw_count == 3, "raw exact occurrences"),
        (normalized_docs == ["A", "E", "F"], "normalized exact documents"),
        (normalized_count == 4, "normalized exact occurrences"),
        (fuzzy_terms == ["毎運業", "海苔業", "かいうんぎょう"], "candidate terms"),
        (fuzzy_docs == ["B", "C", "D"], "candidate documents"),
        (fuzzy_count == 3, "candidate occurrences"),
        (expanded_docs == ["A", "B", "C", "D", "E", "F"], "expanded documents"),
        (expanded_count == 7, "expanded occurrences"),
        ("E" not in raw_docs, "line-break split missed by raw exact search"),
        ("E" in normalized_docs, "line-break split recovered after normalization"),
        ("B" in fuzzy_docs, "OCR substitution recovered as candidate"),
        ("D" in fuzzy_docs, "unrelated near word remains a candidate"),
        ("C" in fuzzy_docs, "kana alias recovered as candidate"),
    ]
    for passed, label in checks:
        if not passed:
            raise AssertionError(label)

    print("Python", sys.version.split()[0], "/ SQLite", sqlite3.sqlite_version)
    for key, body in CORPUS.items():
        print(key + ":", body.replace("\n", " / "))
    print("Raw exact 海運業: documents=" + ",".join(raw_docs),
          "occurrences=" + str(raw_count))
    print("Normalized exact 海運業: documents=" + ",".join(normalized_docs),
          "occurrences=" + str(normalized_count))
    print("Fuzzy candidates:", ", ".join(fuzzy_terms))
    print("Candidate-only: documents=" + ",".join(fuzzy_docs),
          "occurrences=" + str(fuzzy_count))
    print("Expanded: documents=" + ",".join(expanded_docs),
          "occurrences=" + str(expanded_count))
    for key in CORPUS:
        hits = []
        for term in [CANONICAL] + fuzzy_terms:
            n = count(normalized[key], term)
            if n:
                hits.append(f"{term}×{n}")
        print("Hits", key + ":", ", ".join(hits) or "none")
    print("PASS: 14 checks; B recovered, C alias recovered, D needs review, "
          "E recovered only after normalization.")
    db.close()


if __name__ == "__main__":
    main()
