Subversion Repositories LCARS

Compare Revisions

Last modification

Regard whitespace Rev 291 → Rev 292

/trunk/tools/eazytrans/vuh.py
1,3 → 1,5
#!/usr/bin/env python3
 
'''
Created on 2014-10-20
 
6,13 → 8,15
from sys import argv, stderr
from re import findall, DOTALL, IGNORECASE, match, sub, compile, \
split
from os import chdir
from os.path import dirname, realpath
from os import chdir, stat
from os.path import dirname, realpath, basename
from collections import OrderedDict
from functools import cmp_to_key
from copy import deepcopy
from collections.abc import MutableSequence
from pickle import dump, load
 
debug_level = 2
dictionary = {}
 
prepositions = {
21,6 → 25,17
"t'": 'of'
}
 
def dmsg(*args, **kwargs):
if not hasattr(kwargs, 'min_level') or kwargs['min_level'] is None:
kwargs['min_level'] = 1
 
if not hasattr(kwargs, 'file'):
kwargs['file'] = stderr
 
if debug_level >= kwargs['min_level']:
del kwargs['min_level']
print(*args, **kwargs)
 
class MutableString2(MutableSequence):
def __init__(self, value=None):
self._values = [str(value)] if value is not None else []
52,10 → 67,23
def insert(self, index, value):
raise NotImplementedError
 
def cli_help():
print('Usage: {0} TEXT...'.format(basename(argv[0])))
 
def load_dictionary(dictionary, dictionary_file):
print('Loading dictionary {0} ...'.format(dictionary_file), end='', file=stderr)
dmsg('Loading dictionary '.format(dictionary_file), end='', min_level=1)
 
chdir(dirname(realpath(__file__)))
 
pickle_file = basename(dictionary_file) + '.pickle'
 
try:
pickle_mtime = stat(pickle_file).st_mtime
except FileNotFoundError:
pickle_mtime = None
 
if pickle_mtime is None or stat(dictionary_file).st_mtime > pickle_mtime:
dmsg('from {0} ...'.format(dictionary_file), end='', min_level=1)
with open(dictionary_file) as f:
keys = "ipa|en|lit|pos|com|tag|ex"
indent = None
68,9 → 96,7
dictionary[phrase] = {}
indent = None
else:
m = match(
r'(?P<indent>\s*)(?P<key>{0}):\s*(?P<value>.+)'.format(keys),
line)
m = match(r'(?P<indent>\s*)(?P<key>{0}):\s*(?P<value>.+)'.format(keys), line)
if m is not None:
indent = m.group("indent")
key = m.group("key")
82,8 → 108,18
if len(m.group("indent")) == len(indent) + 2:
dictionary[phrase][key] += (" " + m.group("continuation"))
 
print(' done ({0} entries).'.format(len(dictionary)), file=stderr)
dmsg('\nSaving pickle {0} ...'.format(pickle_file), end='', min_level=1)
# TODO: Pickle should only contain strings to be small
with open(pickle_file, mode='wb') as f: dump(dictionary, f)
dmsg(' done.', min_level=1)
else:
dmsg('from {0} ...'.format(pickle_file), end='', min_level=1)
with open(pickle_file, mode='rb') as f: pickle = load(f)
for key, value in pickle.items():
dictionary[key] = value
 
dmsg(' done ({0} entries).'.format(len(dictionary)), min_level=1)
 
def clean_dictionary(dictionary):
braces_re = compile(r'^\s*\{(.+)\}\s*$')
semicolon_re = compile(r'\s*;\s*')
137,30 → 173,21
 
return cmp_to_key(sort_dict_alnum_vulcan)
 
def translate (word, recursion=False):
translation = dictionary.get(word.lower(), None)
def translate (phrase):
translation = dictionary.get(phrase.lower(), None)
if translation is not None:
translation = translation["en"]
if match('[A-Z]', word):
return sub('[a-z]', lambda ch: ch.group(0).upper(), str(translation), count=1)
return translation
 
if not recursion:
# prepositions attached?
for prep, prep_transl in prepositions.items():
if (match(prep, word)):
real_word = word.replace(r'^' + prep, '')
real_word_transl = translate(real_word, recursion=True)
if real_word_transl is not None:
return prep_transl + ' ' + real_word_transl
 
if recursion:
return None
else:
# Not in dictionary: proper name or missing for other reasons
return '{{{0}}}'.format(word)
 
if __name__ == '__main__':
if len(argv) < 2:
print('Nothing to translate.', end='\n\n', file=stderr)
cli_help()
exit(1)
 
text = argv[1]
 
load_dictionary(dictionary, 'vuh-gol-en.dict.zdb.txt')
clean_dictionary(dictionary)
 
173,23 → 200,39
# except BrokenPipeError:
# pass
 
text = argv[1]
sentences = findall(r'(?!\s+)(?:.+?\.{1,3}|.+$)', text, DOTALL)
dmsg("sentences:", sentences, min_level=2)
for sentence in sentences:
print(sentence)
dmsg("sentence:", sentence, min_level=2)
 
words = findall(r"(?!\s+)[a-z'-]{2,}", sentence, IGNORECASE)
print(words)
clauses = split(r'\s+[-–—]\s+', sentence)
dmsg("clauses:", clauses, min_level=2)
for clause in clauses:
dmsg("clause:", clause, min_level=2)
 
translated_words = list(map(translate, words))
print(translated_words)
words = findall(r'[^\s.]+', clause)
dmsg("words:", words, min_level=2)
 
for index, word in enumerate(words):
sentence = sentence.replace(word, str(translated_words[index]))
print(sentence)
offset = 0
while offset < len(words):
translation = None
 
# replace punctuation
for symbol, replacement in ({" - ": ", "}).items():
sentence = sentence.replace(symbol, replacement)
for i in reversed(range(offset + 1, len(words) + 1)):
phrase = ' '.join(words[offset:i])
 
print(sentence)
dmsg("phrase:", phrase, min_level=2)
 
translation = translate(phrase)
 
if translation is not None:
dmsg("phrase-translation:", translation, min_level=2)
dmsg("words[{0}:{1}] = [\"{2}\"]".format(offset, i, translation), min_level=2)
words[offset:i] = [translation]
offset += i - 1
break
 
if translation is None:
dmsg("phrase-translation:", translation, min_level=2)
offset += 1
 
dmsg("words-translation:", words, min_level=2)