"""Executed data used by both the article and video. No illustrative numeric outputs."""
import json
import re
from collections import Counter
from pathlib import Path

corpus = ['我喜欢机器学习', '机器学习可以解决分类问题', '我正在学习自然语言处理']
special_tokens = ['<PAD>', '<UNK>', '<BOS>', '<EOS>']

def tokenize(text):
    return list(re.sub(r'\s+', '', text.strip().lower()))

def build_vocab(corpus, min_frequency=1):
    counter = Counter(token for text in corpus for token in tokenize(text))
    return special_tokens + [token for token, count in sorted(counter.items()) if count >= min_frequency]

def corpus_report(corpus):
    tokenized = [tokenize(text) for text in corpus]
    lengths = [len(tokens) for tokens in tokenized]
    counter = Counter(token for tokens in tokenized for token in tokens)
    return dict(documents=len(corpus), total_tokens=sum(lengths), unique_tokens=len(counter),
                min_length=min(lengths, default=0), max_length=max(lengths, default=0),
                average_length=sum(lengths)/len(lengths) if lengths else 0,
                empty_documents=sum(length == 0 for length in lengths),
                top_tokens=counter.most_common(10))

vocab = build_vocab(corpus)
mapping = {token: index for index, token in enumerate(vocab)}
data = dict(corpus=corpus, tokens=[tokenize(text) for text in corpus], vocab=vocab,
            vocab2=build_vocab(corpus, 2), mapping=mapping,
            encoded=[mapping.get(token, 1) for token in tokenize(corpus[0])],
            mixed=tokenize('我 喜欢 AI！'), unknown=[mapping.get(token, 1) for token in tokenize('你好')],
            report=corpus_report(corpus), empty_report=corpus_report([]))
assert data['encoded'] == [14, 9, 16, 15, 10, 13, 4]
assert data['report']['total_tokens'] == 30
assert data['report']['unique_tokens'] == 23
assert Counter(tokenize(''.join(corpus)))['学'] == 3
assert len(data['vocab2']) == 9
assert len(tokenize('deep learning is fun')) == 17
Path('public').mkdir(exist_ok=True)
Path('public/example-results.json').write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf8')
print(json.dumps(data, ensure_ascii=False, indent=2))