"""Inspect character TF-IDF using only the bundled synthetic training messages."""
from pathlib import Path
from datetime import datetime,timezone,timedelta
from collections import Counter
import argparse
import hashlib
import json
import math
import platform

import numpy as np
import scipy
import sklearn
from sklearn.feature_extraction.text import TfidfVectorizer

def run(data_path,location='local CPU'):
    raw=data_path.read_bytes();data=json.loads(raw);texts=[text for text,_ in data['train']]
    vectorizer=TfidfVectorizer(analyzer='char',ngram_range=(1,3))
    vectorizer.fit(texts)
    query='ログインできません';analyze=vectorizer.build_analyzer()
    counts=Counter(analyze(query));row=vectorizer.transform([query]).toarray()[0]
    features=vectorizer.get_feature_names_out();documents=[set(analyze(text)) for text in texts]
    raw_weights=np.array([counts[t]*(math.log((1+len(texts))/(1+sum(t in d for d in documents)))+1) for t in features])
    magnitude=float(np.linalg.norm(raw_weights))
    np.testing.assert_allclose(raw_weights/magnitude,row,rtol=1e-12,atol=1e-14)
    terms=[]
    for term in ['ログイ','グイン','ログ','料金','ログイン']:
        index=vectorizer.vocabulary_.get(term)
        terms.append({'term':term,'inVocabulary':index is not None,'queryCount':counts[term],
                      'documentFrequency':None if index is None else sum(term in d for d in documents),
                      'idf':None if index is None else float(vectorizer.idf_[index]),
                      'beforeNormalization':None if index is None else float(raw_weights[index]),
                      'weight':None if index is None else float(row[index])})
    unseen='🦊🪐';unseen_vector=vectorizer.transform([unseen])
    range4=TfidfVectorizer(analyzer='char',ngram_range=(1,4));range4.fit(texts)
    return {'completedAt':datetime.now(timezone.utc).isoformat(),'measurementDateJST':datetime.now(timezone(timedelta(hours=9))).date().isoformat(),'location':location,'platform':platform.system(),
            'python':platform.python_version(),'versions':{'numpy':np.__version__,'scipy':scipy.__version__,'sklearn':sklearn.__version__},
            'scriptSha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),'datasetSha256':hashlib.sha256(raw).hexdigest(),
            'fitSplit':'train','fitCount':len(texts),'classifierTrained':False,'testDataUsedForFit':False,
            'parameters':{'analyzer':'char','ngram_range':[1,3],'norm':'l2','smooth_idf':True,'sublinear_tf':False,'lowercase':True},
            'vocabularySize':len(features),'query':query,'queryNgrams':analyze(query),'terms':terms,
            'rawL2Norm':magnitude,'normalizedL2Norm':float(np.linalg.norm(row)),'sumOfWeights':float(row.sum()),
            'allNonzeroFeatures':[{'term':str(t),'weight':float(row[i])} for i,t in enumerate(features) if row[i]!=0],
            'manualFormulaMatchesLibrary':True,'loginNgrams':analyze('ログイン'),
            'unseen':{'text':unseen,'ngrams':analyze(unseen),'nonzeroFeatures':int(unseen_vector.nnz),'l2Norm':float(np.linalg.norm(unseen_vector.toarray()))},
            'rangeComparison':[{'ngram_range':[1,3],'vocabularySize':len(features),'loginInVocabulary':'ログイン' in vectorizer.vocabulary_},
                               {'ngram_range':[1,4],'vocabularySize':len(range4.vocabulary_),'loginInVocabulary':'ログイン' in range4.vocabulary_}]}

if __name__=='__main__':
    parser=argparse.ArgumentParser();parser.add_argument('--data',default=str(Path(__file__).with_name('dataset.json')));parser.add_argument('--out',default='tfidf-results-local.json');parser.add_argument('--location',default='local CPU');args=parser.parse_args()
    result=run(Path(args.data),args.location);Path(args.out).write_text(json.dumps(result,ensure_ascii=False,indent=2,allow_nan=False)+'\n')
    print(json.dumps({'saved':args.out,'vocabularySize':result['vocabularySize'],'terms':result['terms'],'unseen':result['unseen'],'rangeComparison':result['rangeComparison']},ensure_ascii=False))