import natural from 'natural';
import sw from 'stopword';

const NUM_WORDS = 50;
const MIN_WORD_LENGTH = 3;
const WEIGHT_UPPER = 5;

export default function analyze(text) {
    const wordTokenizer = new natural.WordTokenizer();
    const sentenceTokenizer = new natural.SentenceTokenizer();

    text = removeNewlines(text);

    let sentences = sentenceTokenizer.tokenize(text)
        .map((sentence, id) => ({
            id: id, 
            string: sentence
        }));

    let stemMap = {};

    for (let sentence of sentences) {
        let words = wordTokenizer.tokenize(sentence.string)
            .filter(isRelevantToken)
            .map(token => ({
                word: token,
                stem: getStem(token)
            }));

        for (let entry of words) {
            addToStemMap(stemMap, entry, sentence.id);
        }
    }

    if (Object.keys(stemMap).length < NUM_WORDS) {
        throw new Error(`The text is not long enough. Please provide a text with at least ${NUM_WORDS} different words.`);
    }

    stemMap = filterMostFrequentWords(stemMap, NUM_WORDS);
    stemMap = determineRepresentatives(stemMap);

    let wordArray = Object.values(stemMap);

    let similarities = calculateSimilarities(wordArray);
    
    wordArray = calculateWeights(wordArray);

    return { words: wordArray, similarities: similarities };
}

async function readInput(filename) {
    let text = await fs.readFile(filename, "UTF8");
    text = removeNewlines(text);
    return text;
}

// Filter out stopwords, numbers and short words
function isRelevantToken(word) {
    return (!isStopword(word) && !isNumber(word) && (word.length >= MIN_WORD_LENGTH));
}

function getStem(word) {
    return natural.PorterStemmer.stem(word);
}

// Increase counter of stem of the word, add word to variants and save sentence it occurs in
function addToStemMap(stemMap, entry, sentenceId) {
    let stem = entry.stem;
    let word = entry.word;

    stemMap[stem] ??= {};
    stemMap[stem].count = (stemMap[stem].count ?? 0) + 1;

    stemMap[stem].variants ??= {};
    stemMap[stem].variants[word] = (stemMap[stem].variants[word] ?? 0) + 1;

    stemMap[stem].occurrences ??= [];
    stemMap[stem].occurrences.push(sentenceId);

    return stemMap;
}

function filterMostFrequentWords(ranks, numberOfWords) {
    let arr = Object.entries(ranks)
        .sort((a,b) => { return b[1].count - a[1].count })
        .slice(0, numberOfWords);
    return Object.fromEntries(arr);
}

function determineRepresentatives(ranks) {
    for (let stem in ranks) {
        let rep = getMostFrequentVariant(ranks[stem].variants);
        ranks[stem].representative = rep;
    }
    return ranks;
}

function getMostFrequentVariant(variants) {
    let arr = Object.entries(variants)
        .sort((a,b) => { return b[1] - a[1] });
    return arr[0][0];
}

function calculateSimilarities(words) {
    let similarities = [];
    for (let i = 0; i < NUM_WORDS; i++) {
        for (let j = i+1; j < NUM_WORDS; j++) {
            similarities.push({ word1: i, word2: j, similarity: calculateSimilarity(words[i], words[j]) });
        }
    }
    return similarities;
}

function calculateSimilarity(word1, word2) {
    // Jaccard Similarity
    let intersection = new Set(word1.occurrences.filter(s => word2.occurrences.includes(s)));
    let union = new Set([...word1.occurrences, ...word2.occurrences]);
    return intersection.size / union.size;
}

function calculateWeights(words) {
    const max = words[0].count;
    const min = words[words.length-1].count;
    for (let i = 0; i < words.length; i++) {
        const count = words[i].count;
        const num = Math.log(count) - Math.log(min);
        const den = (max > min ? Math.log(max) - Math.log(min) : 1.0);
        words[i].weight = 1.0 + (WEIGHT_UPPER - 1.0) * num/den;
        words[i].id = i;
    }
    return words;
}

function removeNewlines(string) {
    return string.replace(/\r?\n|\r/g, " ");
}

function isNumber(string) {
    return /^\d+$/.test(string);
}

function isStopword(string) {
    return sw.en.includes(string.toLowerCase());
}