added common_words

Author: mircealungu

.gitignore

@@ -3,4 +3,5 @@
 build
 dist
 *.egg-info
-wordinfo.db
+wordinfo.db
+venv/*

example.py

@@ -0,0 +1,34 @@
+from wordstats import LanguageInfo
+from wordstats.common_words import common_words
+
+French = LanguageInfo.load("fr")
+
+
+print("Top 10 most used words in French")
+print(French.all_words()[:10])
+print(" ")
+
+
+print("Info about the word 'jamais'")
+jamais_info = French.get("jamais")
+print(jamais_info)
+print(" ")
+
+
+print("First 10 common words of more than 10 letters common between French and English")
+
+English = LanguageInfo.load("en")
+count = 0
+for each in French.all_words():
+    if len(each) > 10 and each in English.all_words():
+        print(each)
+        count += 1
+    if count == 10:
+        break
+print("")
+
+
+print("Words (>8chars) common in all the languages*")
+for each in common_words():
+    if len(each) > 8:
+        print(each)

wordstats/common_words.py

@@ -0,0 +1,48 @@
+from wordstats.config import DATA_COMMON_FOLDER
+
+COMMON_WORDS = []
+
+common_words_file_name = DATA_COMMON_FOLDER + "/common.txt"
+
+
+def common_words():
+    global COMMON_WORDS
+
+    if COMMON_WORDS:
+        return COMMON_WORDS
+
+    with open(common_words_file_name) as common_words_file:
+        words_list = common_words_file.read().splitlines()
+        COMMON_WORDS = words_list
+        return COMMON_WORDS
+
+
+def write_common_words_file():
+    """
+    Compute common words between all_languages_with_latin_characters (i.e.
+        "da","de","en","es","fr","it","nl","no","pl","pt","ro")
+    Write them to file
+    :return:
+    """
+    from wordstats import LanguageInfo
+    from wordstats.language_codes import all_languages_with_latin_characters
+
+    common_words = set()
+
+    reference_language = LanguageInfo.load(all_languages_with_latin_characters[0])
+    all_other_languages = [LanguageInfo.load(each) for each in all_languages_with_latin_characters[1:]]
+
+    with open(common_words_file_name, 'w') as common_words_file:
+
+        for each in reference_language.all_words():
+            at_least_one_exception = False
+            for other_language in all_other_languages:
+                if each not in other_language.all_words():
+                    at_least_one_exception = True
+                    continue
+
+            if not at_least_one_exception:
+                if each not in common_words:
+                    common_words.add(each)
+                    common_words_file.write(each + "\n")
+                    print(each)

wordstats/config.py

@@ -1,10 +1,23 @@
-db_uri = 'sqlite:///wordinfo.db'
+import os
+
+import platform
+import tempfile
+import logging as log
+
+tempdir = "/tmp" if platform.system() == "Darwin" else tempfile.gettempdir()
+
+db_uri = 'sqlite:///' + tempdir  + '/wordinfo.db'
+
+log.debug("running with DB URI: " + db_uri )
 
 # for mysql, we want to declare the default character encoding
 # for comm. with the db
 if db_uri.startswith("mysql"):
     db_uri += '?charset=utf8'
 
-DATA_FOLDER= 'language_data/hermitdave/2016'
+DATA_HERMIT_FOLDER = 'language_data/hermitdave/2016'
+
+package_directory = os.path.dirname(os.path.abspath(__file__))
+DATA_COMMON_FOLDER = package_directory + os.sep +'language_data/common'
 
 MAX_WORDS = 10000

wordstats/language_codes.py

@@ -0,0 +1,3 @@
+all_languages = ["da", "de", "el", "en", "es", "fr", "it", "nl", "no", "pl", "pt", "ro", "zh-CN"]
+
+all_languages_with_latin_characters = ["da", "de", "en", "es", "fr", "it", "nl", "no", "pl", "pt", "ro"]

wordstats/language_data/common/Readme.md

@@ -9,13 +9,29 @@
 from .base_service import BaseService, Base
 from .config import MAX_WORDS
 from .metrics_computers import *
+import logging as log
 
 
 class LanguageInfo(object):
     def __init__(self, language_id):
         self.language_id = language_id
         self.word_info_dict = dict()
 
+    @classmethod
+    def load(cls, language_code):
+        from wordstats.loading_from_hermit import load_language_from_hermit
+
+        log.info(f"loading {language_code} from DB")
+        lang = LanguageInfo.load_from_db(language_code)
+
+        if len(lang.all_words()) == 0:
+            log.info(f"loading {language_code} from file")
+            lang = load_language_from_hermit(language_code)
+            log.info(f"caching {language_code} to DB")
+            lang.cache_to_db()
+
+        return lang
+
     def get(self, word):
         """
 

wordstats/language_data/common/find_common.py

@@ -1,19 +1,19 @@
 # "hermit dave" has a nice repository of word frequencies
 # computed for many languages based on movie subtitles
 from .language_info import LanguageInfo
-from .config import DATA_FOLDER
+from .config import DATA_HERMIT_FOLDER
 
 
 def path_of_hermit_language_file(language):
-    file_name = "{0}/{1}/{1}_50k.txt".format(DATA_FOLDER, language)
+    file_name = "{0}/{1}/{1}_50k.txt".format(DATA_HERMIT_FOLDER, language)
     return file_name
 
 
 def load_language_from_hermit(language, hermit_root_folder=None):
 
     # by default use the hermit folder in the config file
     if not hermit_root_folder:
-        hermit_root_folder = DATA_FOLDER
+        hermit_root_folder = DATA_HERMIT_FOLDER
 
     file_name = "{0}/{1}/{1}_50k.txt".format(hermit_root_folder, language)
     d = LanguageInfo.load_from_file(file_name, language)
@@ -24,7 +24,7 @@ def load_multiple_languages_from_hermit(languages, hermit_root_folder=None):
 
     # by default use the hermit folder in the config file
     if not hermit_root_folder:
-        hermit_root_folder = DATA_FOLDER
+        hermit_root_folder = DATA_HERMIT_FOLDER
 
     result = dict()
     for language in languages:

wordstats/language_info.py

@@ -46,7 +46,7 @@ def __init__(self, word, language_id, frequency, difficulty, importance, rank, k
         self.rank = rank
 
     def __str__(self):
-        result = "info: {0} ({1}, freq: {2}, imp: {3}, diff: {4}, rank: {5}, klevel: {6})".format(
+        result = "{0}: (lang: {1}, rank: {5}, freq: {2}, imp: {3}, diff: {4}, klevel: {6})".format(
             self.word,
             self.language_id,
             self.frequency,
@@ -55,7 +55,6 @@ def __str__(self):
             self.rank,
             self.klevel)
 
-        result = result.encode(stdout.encoding)
         return result
 
     @classmethod