WebSVN - LCARS - Diff - Rev 291 and 292 - /trunk/tools/eazytrans/vuh.py

+#!/usr/bin/env python3
 '''
 Created on 2014-10-20
 @author: Thomas 'PointedEars' Lahn <mail@PointedEars.de>
 '''
 from sys import argv, stderr
 from re import findall, DOTALL, IGNORECASE, match, sub, compile, \
     split
-from os import chdir
+from os import chdir, stat
-from os.path import dirname, realpath
+from os.path import dirname, realpath, basename
 from collections import OrderedDict
 from functools import cmp_to_key
 from copy import deepcopy
 from collections.abc import MutableSequence
+from pickle import dump, load
+debug_level = 2
 dictionary = {}
 prepositions = {
     "fi'": 'on',
     "na'": 'at|to',
     "t'": 'of'
+}
+def dmsg(*args, **kwargs):
+    if not hasattr(kwargs, 'min_level') or kwargs['min_level'] is None:
+        kwargs['min_level'] = 1
+    if not hasattr(kwargs, 'file'):
+        kwargs['file'] = stderr
+    if debug_level >= kwargs['min_level']:
+        del kwargs['min_level']
+        print(*args, **kwargs)
 class MutableString2(MutableSequence):
     def __init__(self, value=None):
         self._values = [str(value)] if value is not None else []
     def __add__(self, value):
         self._values.append(values)
     def insert(self, index, value):
         raise NotImplementedError
+def cli_help():
+    print('Usage: {0} TEXT...'.format(basename(argv[0])))
 def load_dictionary(dictionary, dictionary_file):
-    print('Loading dictionary {0} ...'.format(dictionary_file), end='', file=stderr)
+    dmsg('Loading dictionary '.format(dictionary_file), end='', min_level=1)
     chdir(dirname(realpath(__file__)))
-    with open(dictionary_file) as f:
-        keys = "ipa|en|lit|pos|com|tag|ex"
-        indent = None
-        value = None
+    pickle_file = basename(dictionary_file) + '.pickle'
-        for line in f:
+    try:
-            m = match(r'^\s*vuh:\s*(?P<phrase>.+)', line)
+        pickle_mtime = stat(pickle_file).st_mtime
+    except FileNotFoundError:
-            if m is not None:
+        pickle_mtime = None
+    if pickle_mtime is None or stat(dictionary_file).st_mtime > pickle_mtime:
+        dmsg('from {0} ...'.format(dictionary_file), end='', min_level=1)
-                phrase = m.group("phrase")
+        with open(dictionary_file) as f:
-                dictionary[phrase] = {}
+            keys = "ipa|en|lit|pos|com|tag|ex"
-                indent = None
+            indent = None
-            else:
+            value = None
-                m = match(
+            for line in f:
-                    r'(?P<indent>\s*)(?P<key>{0}):\s*(?P<value>.+)'.format(keys),
+                m = match(r'^\s*vuh:\s*(?P<phrase>.+)', line)
-                    line)
                 if m is not None:
-                    indent = m.group("indent")
+                    phrase = m.group("phrase")
-                    key = m.group("key")
+                    dictionary[phrase] = {}
-                    value = m.group("value")
+                    indent = None
-                    value = dictionary[phrase][key] = MutableString2(value)
-                elif indent is not None:
+                else:
-                    m = match(r'(?P<indent>\s+)(?P<continuation>\S.*)', line)
+                    m = match(r'(?P<indent>\s*)(?P<key>{0}):\s*(?P<value>.+)'.format(keys), line)
                     if m is not None:
+                        indent = m.group("indent")
+                        key = m.group("key")
+                        value = m.group("value")
+                        value = dictionary[phrase][key] = MutableString2(value)
+                    elif indent is not None:
+                        m = match(r'(?P<indent>\s+)(?P<continuation>\S.*)', line)
+                        if m is not None:
-                        if len(m.group("indent")) == len(indent) + 2:
+                            if len(m.group("indent")) == len(indent) + 2:
-                            dictionary[phrase][key] += (" " + m.group("continuation"))
+                                dictionary[phrase][key] += (" " + m.group("continuation"))
+        dmsg('\nSaving pickle {0} ...'.format(pickle_file), end='', min_level=1)
+        # TODO: Pickle should only contain strings to be small
+        with open(pickle_file, mode='wb') as f: dump(dictionary, f)
+        dmsg(' done.', min_level=1)
+    else:
+        dmsg('from {0} ...'.format(pickle_file), end='', min_level=1)
+        with open(pickle_file, mode='rb') as f: pickle = load(f)
+        for key, value in pickle.items():
+            dictionary[key] = value
-    print(' done ({0} entries).'.format(len(dictionary)), file=stderr)
+    dmsg(' done ({0} entries).'.format(len(dictionary)), min_level=1)
 def clean_dictionary(dictionary):
     braces_re = compile(r'^\s*\{(.+)\}\s*$')
     semicolon_re = compile(r'\s*;\s*')
         return 1 if len(b) < len(a) else 0
     return cmp_to_key(sort_dict_alnum_vulcan)
-def translate (word, recursion=False):
+def translate (phrase):
-    translation = dictionary.get(word.lower(), None)
+    translation = dictionary.get(phrase.lower(), None)
     if translation is not None:
-        translation = translation["en"]
-        if match('[A-Z]', word):
-            return sub('[a-z]', lambda ch: ch.group(0).upper(), str(translation), count=1)
         return translation
-    if not recursion:
-        # prepositions attached?
-        for prep, prep_transl in prepositions.items():
-            if (match(prep, word)):
-                real_word = word.replace(r'^' + prep, '')
-                real_word_transl = translate(real_word, recursion=True)
-                if real_word_transl is not None:
-                    return prep_transl + ' ' + real_word_transl
-    if recursion:
-        return None
+    return None
-    else:
-        # Not in dictionary: proper name or missing for other reasons
-        return '{{{0}}}'.format(word)
 if __name__ == '__main__':
+    if len(argv) < 2:
+        print('Nothing to translate.', end='\n\n', file=stderr)
+        cli_help()
+        exit(1)
+    text = argv[1]
     load_dictionary(dictionary, 'vuh-gol-en.dict.zdb.txt')
     clean_dictionary(dictionary)
 #     try:
 #         for phrase, data in OrderedDict(sorted(
 #         )).items():
 #             print(phrase, "=", data)
 #     except BrokenPipeError:
 #         pass
-    text = argv[1]
     sentences = findall(r'(?!\s+)(?:.+?\.{1,3}|.+$)', text, DOTALL)
+    dmsg("sentences:", sentences, min_level=2)
     for sentence in sentences:
+        dmsg("sentence:", sentence, min_level=2)
+        clauses = split(r'\s+[-–—]\s+', sentence)
+        dmsg("clauses:", clauses, min_level=2)
+        for clause in clauses:
+            dmsg("clause:", clause, min_level=2)
+            words = findall(r'[^\s.]+', clause)
+            dmsg("words:", words, min_level=2)
-        print(sentence)
+            offset = 0
+            while offset < len(words):
+                translation = None
+                for i in reversed(range(offset + 1, len(words) + 1)):
+                    phrase = ' '.join(words[offset:i])
-        words = findall(r"(?!\s+)[a-z'-]{2,}", sentence, IGNORECASE)
+                    dmsg("phrase:", phrase, min_level=2)
-        print(words)
-        translated_words = list(map(translate, words))
+                    translation = translate(phrase)
-        print(translated_words)
-        for index, word in enumerate(words):
+                    if translation is not None:
-            sentence = sentence.replace(word, str(translated_words[index]))
+                        dmsg("phrase-translation:", translation, min_level=2)
+                        dmsg("words[{0}:{1}] = [\"{2}\"]".format(offset, i, translation), min_level=2)
+                        words[offset:i] = [translation]
+                        offset += i - 1
-        print(sentence)
+                        break
-        # replace punctuation
+                if translation is None:
-        for symbol, replacement in ({" - ": ", "}).items():
+                    dmsg("phrase-translation:", translation, min_level=2)
-            sentence = sentence.replace(symbol, replacement)
+                    offset += 1
-        print(sentence)
+            dmsg("words-translation:", words, min_level=2)

Subversion Repositories LCARS

(root)/trunk/tools/eazytrans/vuh.py - Rev 291 → 292