"""Free CPU lexical baseline; does not reproduce EmbeddingGemma hybrid scores.

pip install numpy==2.3.5 rank-bm25==0.2.2 PyYAML==6.0.2
python reproduce_bm25.py --cache ./source-cache --output ./baseline-results
"""
import argparse
import hashlib
import importlib.metadata
import json
import math
import platform
import re
import sqlite3
import urllib.request
from pathlib import Path
from datetime import datetime, timezone

import yaml
from rank_bm25 import BM25Okapi

REV = 'f83ff7abf531573227e5e1cf12c7a116bd69e447'
BASE = f'https://raw.githubusercontent.com/ns-0437/relevant-section-identification/{REV}/'
EXPECTED = {'chroma/chroma.sqlite3':'237641e90d2262f4b3b79252b4d10dc45e0d5a11dc63475f0c5ba0056e6a8b1a','evaluation/eval_gold.yaml':'fccb4ae541910725f3568bb939b485421caeed25f95e5c227af5aa28d67f552d'}
STOP = set('a an and are as at be by for from has have how in is it its of on or that the this to was what when where which who why will with does do did can should would there these those'.split())

def tokenize(text):
    return [t for t in re.findall(r'[a-z0-9_]+', (text or '').lower()) if len(t)>1 and t not in STOP]

def metrics(pages, gold):
    top = pages[:5]
    dcg = sum(gold.get(p, 0) / math.log2(i+2) for i,p in enumerate(top))
    ideal = sum(g / math.log2(i+2) for i,g in enumerate(sorted(gold.values(), reverse=True)[:5]))
    return {'hit_at_5': int(bool(set(top)&set(gold))), 'recall_at_5': len(set(top)&set(gold))/len(gold), 'ndcg_at_5': dcg/ideal if ideal else 0}

def main():
    ap=argparse.ArgumentParser()
    ap.add_argument('--cache', type=Path, default=Path('source-cache'))
    ap.add_argument('--output', type=Path, default=Path('baseline-results'))
    args=ap.parse_args()
    args.cache.mkdir(parents=True, exist_ok=True)
    args.output.mkdir(parents=True, exist_ok=True)
    files={'chroma.sqlite3':'chroma/chroma.sqlite3', 'eval_gold.yaml':'evaluation/eval_gold.yaml'}
    hashes={}
    for name, remote in files.items():
        p=args.cache/name
        if not p.exists():
            urllib.request.urlretrieve(BASE+remote,p)
        hashes[remote]=hashlib.sha256(p.read_bytes()).hexdigest()
        if hashes[remote] != EXPECTED[remote]:
            raise ValueError(f'Source checksum mismatch: {remote}')
    db=sqlite3.connect((args.cache/'chroma.sqlite3').resolve().as_uri()+'?mode=ro',uri=True)
    assert db.execute('pragma integrity_check').fetchone()[0]=='ok'
    segment=db.execute("select s.id from segments s join collections c on s.collection=c.id where c.name=? and s.scope='METADATA'",('max77751_v3',)).fetchone()[0]
    chunks=[]
    for row_id, chunk_id in db.execute('select id,embedding_id from embeddings where segment_id=? order by id',(segment,)):
        meta={k: next((v for v in values if v is not None),None) for k,*values in db.execute('select key,string_value,int_value,float_value,bool_value from embedding_metadata where id=?',(row_id,))}
        chunks.append({'id':chunk_id,'page':int(meta['page_number']),'text':f"{meta.get('title','')} {meta['chroma:document']}"})
    bm25=BM25Okapi([tokenize(c['text']) for c in chunks])
    queries=yaml.safe_load((args.cache/'eval_gold.yaml').read_text(encoding='utf-8'))['queries']
    results=[]
    for n,q in enumerate(queries,1):
        scores=bm25.get_scores(tokenize(q['query']))
        order=sorted(range(len(chunks)),key=lambda i:-scores[i])[:104]
        pages=list(dict.fromkeys(chunks[i]['page'] for i in order))
        gold={int(k):v for k,v in q['relevant'].items()}
        results.append({'id':n,'query':q['query'],'kind':q['kind'],'split':'holdout' if q.get('holdout',False) else 'development','gold':gold,'ranked_pages':pages,'ranked_chunks':[{'id':chunks[i]['id'],'page':chunks[i]['page'],'bm25_score':float(scores[i])} for i in order],**metrics(pages,gold)})
    summary={}
    for split in ['all','development','holdout']:
        rows=[r for r in results if split=='all' or r['split']==split]
        summary[split]={'n':len(rows),**{k:sum(r[k] for r in rows)/len(rows) for k in ['hit_at_5','recall_at_5','ndcg_at_5']}}
    manifest={'source_revision':REV,'run_utc':datetime.now(timezone.utc).isoformat(),'method':'BM25Okapi lexical baseline; no dense model, no section expansion','collection':'max77751_v3','chunks':len(chunks),'candidate_chunks':104,'tie_break':'SQLite embedding row id ascending; may differ from Chroma API ordering','parameters':{'k1':1.5,'b':0.75,'epsilon':0.25},'python':platform.python_version(),'packages':{p:importlib.metadata.version(p) for p in ['numpy','rank-bm25','PyYAML']},'source_sha256':hashes,'summary':summary,'limitations':['Single document and author-created relevance judgments.','Existing public split retained; not a fresh independent holdout.','Extraction and embedding pipeline were not rerun.','Not a replication of the reported hybrid + section nDCG 0.728.']}
    (args.output/'bm25-results.json').write_text(json.dumps(results,indent=2)+'\n',encoding='utf-8')
    (args.output/'bm25-manifest.json').write_text(json.dumps(manifest,indent=2)+'\n',encoding='utf-8')
    print(json.dumps(manifest,indent=2))

if __name__=='__main__':
    main()