import re
import unicodedata

_PAGE_NUMBER_RE = re.compile(r"^\s*(page\s*)?\d{1,4}(\s*(of|/)\s*\d{1,4})?\s*$", re.IGNORECASE)
_COPYRIGHT_RE = re.compile(r"copyright|^\s*©|all rights reserved", re.IGNORECASE)
_REFERENCES_HEADING_RE = re.compile(r"^\s*(references|bibliography|works cited)\s*:?\s*$", re.IGNORECASE)
_HYPHEN_BREAK_RE = re.compile(r"(\w)-\n(\w)")
_REPEATED_PUNCT_RE = re.compile(r"([.\-_=*])\1{4,}")
_MULTI_BLANK_RE = re.compile(r"\n{3,}")
_TRAILING_WS_RE = re.compile(r"[ \t]+\n")


def clean_text(raw_text: str) -> tuple[str, dict]:
    """Runs the Module 3 cleaning pipeline. Returns (cleaned_text, report)."""
    report = {
        "page_numbers_removed": 0,
        "copyright_lines_removed": 0,
        "duplicate_paragraphs_removed": 0,
        "references_section_removed": False,
        "ocr_hyphen_fixes": 0,
    }

    text = unicodedata.normalize("NFKC", raw_text)

    hyphen_fixes = len(_HYPHEN_BREAK_RE.findall(text))
    text = _HYPHEN_BREAK_RE.sub(r"\1\2", text)
    report["ocr_hyphen_fixes"] = hyphen_fixes

    text = _REPEATED_PUNCT_RE.sub(lambda m: m.group(1) * 3, text)

    lines = text.split("\n")
    kept_lines: list[str] = []
    references_hit = False
    for line in lines:
        if references_hit:
            continue
        if _REFERENCES_HEADING_RE.match(line):
            references_hit = True
            report["references_section_removed"] = True
            continue
        if _PAGE_NUMBER_RE.match(line) and line.strip():
            report["page_numbers_removed"] += 1
            continue
        if _COPYRIGHT_RE.search(line):
            report["copyright_lines_removed"] += 1
            continue
        kept_lines.append(line)
    text = "\n".join(kept_lines)

    paragraphs = re.split(r"\n\s*\n", text)
    seen: set[str] = set()
    deduped: list[str] = []
    for para in paragraphs:
        normalized = re.sub(r"\s+", " ", para).strip().lower()
        if not normalized:
            continue
        if normalized in seen:
            report["duplicate_paragraphs_removed"] += 1
            continue
        seen.add(normalized)
        deduped.append(para.strip())
    text = "\n\n".join(deduped)

    text = _TRAILING_WS_RE.sub("\n", text)
    text = _MULTI_BLANK_RE.sub("\n\n", text)
    text = text.strip()

    return text, report