Creutz and Lagus (2002)

[LinguList]

Creutz and Lagus (2002) has been in part implemented by myself, but not yet adjusted to the new structure.

[LinguList]

from lingpy import *
import math
from collections import defaultdict

def model_cost_mdl(words, k=5):
    
    scores = defaultdict(int)
    for word in words:
        for morph in word:
            scores[morph] += 1

    source_costs = []
    code_costs = []
    for word in words:
        for morph in word:
            source_cost = -1 * math.log(scores[morph] / sum(scores.values()))
            source_costs += [source_cost]
    for morph, freqs in scores.items():
        code_cost = k * len(morph)
        code_costs += [code_cost]
    print("a", sum(source_costs))
    print("b", sum(code_costs))
    return sum(source_costs) + sum(code_costs)


a = [["abcde"], ["abc"], ["de"]]
b = [["abc", "de"], ["abc"], ["de"]]
c = [["ab", "c", "de"], ["abc"], ["de"]]
d = [["ab", "c", "de"], ["ab", "c"], ["de"]]


print(model_cost_mdl(a))
print(model_cost_mdl(b))
print(model_cost_mdl(c))
print(model_cost_mdl(d))

[LinguList]

def recursive_search(words, bits=5, gens=20):
    last_words = [w for w in words]
    for i in range(gens):
        new_words = [w for w in last_words]
        old_score = model_cost_mdl(new_words, bits)
        for j, word in enumerate(new_words):
            for k, morpheme in enumerate(word):
                current_words = [w for w in new_words]
                scores = []
                for m_a, m_b in n_splits(morpheme, 2):
                    current_words[j][k] = "".join(m_a)
                    current_words[j].insert(k+1, "".join(m_b))
                    new_score = model_cost_mdl(current_words, bits)
                    scores.append((new_score, k, "".join(m_a), "".join(m_b)))
                    current_words[j][k] = morpheme
                    del current_words[j][k+1]
                if scores:
                    new_score, k, m_a, m_b = sorted(scores, key=lambda x: x[0])[0]
                    if new_score < old_score:
                        new_words[j][k] = m_a
                        new_words[j].insert(k+1, m_b)
                        old_score = new_score
                    new_words[j][k] = current_words[j][k] 
    return new_words