#!/usr/bin/env python3
"""
Crawl the legacy shodh.net journal archive (Vol. 1-17, 25 issues) into an
import manifest + local PDF store.

Source structure (Joomla phocadownload):
  - Issue page:  /index.php?option=com_phocadownload&view=category&id={cat}&Itemid={n}
    Each downloadable file is a table row:
      <a href="...download={fileId}:{slug}...">TITLE (often "N. Title - Author")</a>
      <small>(SIZE)</small>
    optionally followed by a <td colspan="5"> abstract block.
  - The download URL returns a tiny HTML shim whose <script> redirects to the
    real file under /phocadownload/... — we follow that.

Outputs (relative to the api repo root):
  - legacy-import/manifest.json        import manifest (issues -> articles)
  - legacy-import/validation-report.md gaps and oddities, human-readable
  - uploads/archive/vol{V}-{label}/{fileId}.pdf   the PDFs (StorageService keys)

Run from shodh-sanchayan-api/:  python3 scripts/legacy_crawl.py [--no-pdfs]
Re-running is safe: PDFs already present (matching size) are not re-fetched.
"""

import hashlib
import html as htmllib
import json
import re
import sys
import time
import unicodedata
import urllib.parse
import urllib.request
from pathlib import Path

BASE = "https://shodh.net"
DELAY_SECONDS = 0.6
USER_AGENT = "ShodhSanchayanMigration/1.0 (archive import; contact info@shodh.net)"

# category id -> (volume, issue label, Itemid) — from docs/legacy-site/legacy-site-inventory.md
ISSUES = {
    1:  (1, "1", 77),   2:  (1, "2", 78),   36: (2, "1&2", 79),
    37: (3, "1", 95),   40: (3, "2", 98),   41: (4, "1", 100),
    42: (4, "2", 116),  43: (5, "1", 117),  46: (5, "2", 125),
    49: (6, "1", 127),  48: (6, "2", 131),  47: (7, "1&2", 136),
    50: (8, "1", 138),  51: (8, "2", 139),  52: (9, "1", 142),
    54: (9, "2", 143),  55: (10, "1", 145), 56: (10, "2", 179),
    57: (11, "1&2", 183), 58: (12, "1&2", 184), 59: (13, "1&2", 185),
    61: (14, "1&2", 188), 63: (15, "1&2", 192), 64: (16, "1&2", 193),
    65: (17, "1&2", 194),
}

ISSN_PRINT = "0975-1254"
ISSN_ONLINE = "2249-9180"

REPO_ROOT = Path(__file__).resolve().parent.parent
OUT_DIR = REPO_ROOT / "legacy-import"
PDF_ROOT = REPO_ROOT / "uploads"  # matches STORAGE_LOCAL_PATH=./uploads

FILE_ROW_RE = re.compile(
    r'<a href="(/index\.php\?option=com_phocadownload[^"]*?download=(\d+):([^&"]*)[^"]*)"[^>]*>'
    r'([^<]+)</a>\s*<small[^>]*>\(([\d.]+)\s*(kB|KB|MB|B)\)',
)
ABSTRACT_RE = re.compile(r'<td[^>]*colspan="5"[^>]*>(.*?)</td>', re.DOTALL)
REDIRECT_RE = re.compile(r"document\.location\.href='([^']+)'")
DEVANAGARI_RE = re.compile(r"[ऀ-ॿ]")


def fetch(url: str) -> bytes:
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    with urllib.request.urlopen(req, timeout=60) as resp:
        return resp.read()


def polite_fetch(url: str) -> bytes:
    time.sleep(DELAY_SECONDS)
    return fetch(url)


def size_to_bytes(value: str, unit: str) -> int:
    factor = {"b": 1, "kb": 1024, "mb": 1024 * 1024}[unit.lower()]
    return int(float(value) * factor)


def is_hindi(text: str) -> bool:
    return bool(text) and len(DEVANAGARI_RE.findall(text)) > len(text) * 0.2


def clean(text: str) -> str:
    text = htmllib.unescape(re.sub(r"<[^>]+>", " ", text))
    text = unicodedata.normalize("NFC", text)
    return re.sub(r"\s+", " ", text).strip()


def split_title_author(raw: str):
    """'3. Title text - Author Name' -> (order_no, title, author). Best effort."""
    raw = clean(raw)
    order_no = None
    m = re.match(r"^(\d+)\s*[\.\)]\s*(.*)$", raw)
    if m:
        order_no = int(m.group(1))
        raw = m.group(2).strip()
    # author after the LAST dash-like separator (require a following capital/Devanagari word)
    m = re.search(r"\s+[-–—]{1,2}\s*([^-–—]{3,80})$", raw)
    title, author = raw, None
    if m and not m.group(1).strip()[0].isdigit():
        author = m.group(1).strip()
        title = raw[: m.start()].strip().rstrip("-–— ")
    return order_no, title, author


def parse_issue_page(html: str):
    """Yields article dicts in page order, attaching the abstract that follows a row."""
    rows = []
    for row_html in re.split(r"</tr>", html):
        m = FILE_ROW_RE.search(row_html)
        if m:
            _, file_id, slug, raw_title, size_val, size_unit = m.groups()
            rows.append({
                "kind": "file",
                "fileId": int(file_id),
                "slug": urllib.parse.unquote(slug),
                "rawTitle": clean(raw_title),
                "sizeBytes": size_to_bytes(size_val, size_unit),
            })
        else:
            am = ABSTRACT_RE.search(row_html)
            if am:
                text = clean(am.group(1))
                if len(text) > 40:
                    rows.append({"kind": "abstract", "text": text})

    articles, seen = [], set()
    for row in rows:
        if row["kind"] == "file":
            if row["fileId"] in seen:  # title link + Download link produce dup matches
                continue
            seen.add(row["fileId"])
            articles.append(row)
        elif articles and "abstract" not in articles[-1]:
            articles[-1]["abstract"] = row["text"]
    return articles


def main():
    download_pdfs = "--no-pdfs" not in sys.argv
    OUT_DIR.mkdir(exist_ok=True)

    manifest = {"source": BASE, "crawledAt": None, "issnPrint": ISSN_PRINT,
                "issnOnline": ISSN_ONLINE, "issues": []}
    problems = []
    total_articles = total_pdfs = 0

    for sort_order, (cat_id, (volume, label, itemid)) in enumerate(sorted(ISSUES.items(), key=lambda kv: (kv[1][0], kv[1][1]))):
        url = f"{BASE}/index.php?option=com_phocadownload&view=category&id={cat_id}&Itemid={itemid}"
        print(f"[issue] Vol.{volume} Issue {label} (cat {cat_id}) …", flush=True)
        try:
            page = polite_fetch(url).decode("utf-8", errors="replace")
        except Exception as e:
            problems.append(f"- Vol.{volume}/{label}: issue page fetch FAILED: {e}")
            continue
        if "�" in page:
            problems.append(f"- Vol.{volume}/{label}: page contains replacement chars (encoding damage)")

        parsed = parse_issue_page(page)
        if not parsed:
            problems.append(f"- Vol.{volume}/{label}: no downloadable files found on page")

        folder = f"archive/vol{volume}-{label.replace('&', 'and')}"
        issue_entry = {
            "legacyCategoryId": cat_id, "volume": volume, "issueLabel": label,
            "sortOrder": sort_order, "articles": [],
        }

        for position, row in enumerate(parsed, start=1):
            order_no, title, author = split_title_author(row["rawTitle"])
            hindi_title = is_hindi(title)
            abstract = row.get("abstract")
            article = {
                "legacyFileId": row["fileId"],
                "legacySlug": row["slug"],
                "titleHi": title if hindi_title else None,
                "titleEn": None if hindi_title else title,
                "authorsHi": author if (author and is_hindi(author)) else None,
                "authorsEn": author if (author and not is_hindi(author)) else None,
                "abstractHi": abstract if (abstract and is_hindi(abstract)) else None,
                "abstractEn": abstract if (abstract and not is_hindi(abstract)) else None,
                "pageRange": None,
                "articleOrder": order_no if order_no is not None else position,
                "pdfSizeBytes": row["sizeBytes"],
                "pdfKey": None, "sha256": None,
            }

            if download_pdfs:
                pdf_key = f"{folder}/{row['fileId']}.pdf"
                target = PDF_ROOT / pdf_key
                target.parent.mkdir(parents=True, exist_ok=True)
                existing_cover = next((PDF_ROOT / f"{folder}/{row['fileId']}.{e}"
                                       for e in ("jpg", "png")
                                       if (PDF_ROOT / f"{folder}/{row['fileId']}.{e}").exists()), None)
                if existing_cover:
                    article["coverImageKey"] = str(existing_cover.relative_to(PDF_ROOT))
                    if not issue_entry.get("coverImageKey"):
                        issue_entry["coverImageKey"] = article["coverImageKey"]
                elif target.exists() and target.stat().st_size > 1024:
                    article["pdfKey"] = pdf_key
                    article["sha256"] = hashlib.sha256(target.read_bytes()).hexdigest()
                else:
                    shim_url = (f"{BASE}/index.php?option=com_phocadownload&view=category"
                                f"&download={row['fileId']}:{urllib.parse.quote(row['slug'])}"
                                f"&id={cat_id}:x&Itemid={itemid}")
                    try:
                        shim = polite_fetch(shim_url)
                        rm = REDIRECT_RE.search(shim.decode("latin-1", errors="replace"))
                        if rm:
                            file_url = urllib.parse.quote(rm.group(1), safe=":/%")
                            blob = polite_fetch(file_url)
                        elif b"%PDF" in shim[:4096]:
                            # Some legacy rows serve the PDF inline after the HTML head
                            blob = shim[shim.index(b"%PDF"):]
                        else:
                            raise ValueError("no redirect in download shim")

                        if blob.startswith(b"%PDF"):
                            target.write_bytes(blob)
                            article["pdfKey"] = pdf_key
                            article["sha256"] = hashlib.sha256(blob).hexdigest()
                            article["pdfSizeBytes"] = len(blob)
                            total_pdfs += 1
                        elif blob[:3] == b"\xff\xd8\xff" or blob[:8] == b"\x89PNG\r\n\x1a\n":
                            # Issue cover uploaded as an image, not a PDF
                            ext = "jpg" if blob[:3] == b"\xff\xd8\xff" else "png"
                            img_key = f"{folder}/{row['fileId']}.{ext}"
                            (PDF_ROOT / img_key).write_bytes(blob)
                            article["coverImageKey"] = img_key
                            article["pdfSizeBytes"] = len(blob)
                            if not issue_entry.get("coverImageKey"):
                                issue_entry["coverImageKey"] = img_key
                        else:
                            raise ValueError(f"not a PDF ({blob[:12]!r})")
                    except Exception as e:
                        problems.append(f"- Vol.{volume}/{label} file {row['fileId']} "
                                        f"({row['rawTitle'][:60]}): PDF fetch FAILED: {e}")

            issue_entry["articles"].append(article)
            total_articles += 1

        print(f"        {len(issue_entry['articles'])} files", flush=True)
        manifest["issues"].append(issue_entry)

    manifest["crawledAt"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    (OUT_DIR / "manifest.json").write_text(
        json.dumps(manifest, ensure_ascii=False, indent=1), encoding="utf-8")

    report = [
        "# Legacy crawl validation report",
        f"Crawled: {manifest['crawledAt']}  |  Issues: {len(manifest['issues'])}/25  |  "
        f"Files: {total_articles}  |  PDFs fetched this run: {total_pdfs}",
        "",
        "## Problems" if problems else "## Problems: none",
        *problems,
    ]
    (OUT_DIR / "validation-report.md").write_text("\n".join(report), encoding="utf-8")
    print(f"\nDone. {len(manifest['issues'])} issues, {total_articles} files, "
          f"{len(problems)} problem(s). See legacy-import/validation-report.md")


if __name__ == "__main__":
    main()