1,3 → 1,5 |
#!/usr/bin/env python3 |
|
''' |
Created on 2014-10-20 |
|
6,13 → 8,15 |
from sys import argv, stderr |
from re import findall, DOTALL, IGNORECASE, match, sub, compile, \ |
split |
from os import chdir |
from os.path import dirname, realpath |
from os import chdir, stat |
from os.path import dirname, realpath, basename |
from collections import OrderedDict |
from functools import cmp_to_key |
from copy import deepcopy |
from collections.abc import MutableSequence |
from pickle import dump, load |
|
debug_level = 2 |
dictionary = {} |
|
prepositions = { |
21,6 → 25,17 |
"t'": 'of' |
} |
|
def dmsg(*args, **kwargs): |
if not hasattr(kwargs, 'min_level') or kwargs['min_level'] is None: |
kwargs['min_level'] = 1 |
|
if not hasattr(kwargs, 'file'): |
kwargs['file'] = stderr |
|
if debug_level >= kwargs['min_level']: |
del kwargs['min_level'] |
print(*args, **kwargs) |
|
class MutableString2(MutableSequence): |
def __init__(self, value=None): |
self._values = [str(value)] if value is not None else [] |
52,10 → 67,23 |
def insert(self, index, value): |
raise NotImplementedError |
|
def cli_help(): |
print('Usage: {0} TEXT...'.format(basename(argv[0]))) |
|
def load_dictionary(dictionary, dictionary_file): |
print('Loading dictionary {0} ...'.format(dictionary_file), end='', file=stderr) |
dmsg('Loading dictionary '.format(dictionary_file), end='', min_level=1) |
|
chdir(dirname(realpath(__file__))) |
|
pickle_file = basename(dictionary_file) + '.pickle' |
|
try: |
pickle_mtime = stat(pickle_file).st_mtime |
except FileNotFoundError: |
pickle_mtime = None |
|
if pickle_mtime is None or stat(dictionary_file).st_mtime > pickle_mtime: |
dmsg('from {0} ...'.format(dictionary_file), end='', min_level=1) |
with open(dictionary_file) as f: |
keys = "ipa|en|lit|pos|com|tag|ex" |
indent = None |
68,9 → 96,7 |
dictionary[phrase] = {} |
indent = None |
else: |
m = match( |
r'(?P<indent>\s*)(?P<key>{0}):\s*(?P<value>.+)'.format(keys), |
line) |
m = match(r'(?P<indent>\s*)(?P<key>{0}):\s*(?P<value>.+)'.format(keys), line) |
if m is not None: |
indent = m.group("indent") |
key = m.group("key") |
82,8 → 108,18 |
if len(m.group("indent")) == len(indent) + 2: |
dictionary[phrase][key] += (" " + m.group("continuation")) |
|
print(' done ({0} entries).'.format(len(dictionary)), file=stderr) |
dmsg('\nSaving pickle {0} ...'.format(pickle_file), end='', min_level=1) |
# TODO: Pickle should only contain strings to be small |
with open(pickle_file, mode='wb') as f: dump(dictionary, f) |
dmsg(' done.', min_level=1) |
else: |
dmsg('from {0} ...'.format(pickle_file), end='', min_level=1) |
with open(pickle_file, mode='rb') as f: pickle = load(f) |
for key, value in pickle.items(): |
dictionary[key] = value |
|
dmsg(' done ({0} entries).'.format(len(dictionary)), min_level=1) |
|
def clean_dictionary(dictionary): |
braces_re = compile(r'^\s*\{(.+)\}\s*$') |
semicolon_re = compile(r'\s*;\s*') |
137,30 → 173,21 |
|
return cmp_to_key(sort_dict_alnum_vulcan) |
|
def translate (word, recursion=False): |
translation = dictionary.get(word.lower(), None) |
def translate (phrase): |
translation = dictionary.get(phrase.lower(), None) |
if translation is not None: |
translation = translation["en"] |
if match('[A-Z]', word): |
return sub('[a-z]', lambda ch: ch.group(0).upper(), str(translation), count=1) |
return translation |
|
if not recursion: |
# prepositions attached? |
for prep, prep_transl in prepositions.items(): |
if (match(prep, word)): |
real_word = word.replace(r'^' + prep, '') |
real_word_transl = translate(real_word, recursion=True) |
if real_word_transl is not None: |
return prep_transl + ' ' + real_word_transl |
|
if recursion: |
return None |
else: |
# Not in dictionary: proper name or missing for other reasons |
return '{{{0}}}'.format(word) |
|
if __name__ == '__main__': |
if len(argv) < 2: |
print('Nothing to translate.', end='\n\n', file=stderr) |
cli_help() |
exit(1) |
|
text = argv[1] |
|
load_dictionary(dictionary, 'vuh-gol-en.dict.zdb.txt') |
clean_dictionary(dictionary) |
|
173,23 → 200,39 |
# except BrokenPipeError: |
# pass |
|
text = argv[1] |
sentences = findall(r'(?!\s+)(?:.+?\.{1,3}|.+$)', text, DOTALL) |
dmsg("sentences:", sentences, min_level=2) |
for sentence in sentences: |
print(sentence) |
dmsg("sentence:", sentence, min_level=2) |
|
words = findall(r"(?!\s+)[a-z'-]{2,}", sentence, IGNORECASE) |
print(words) |
clauses = split(r'\s+[-–—]\s+', sentence) |
dmsg("clauses:", clauses, min_level=2) |
for clause in clauses: |
dmsg("clause:", clause, min_level=2) |
|
translated_words = list(map(translate, words)) |
print(translated_words) |
words = findall(r'[^\s.]+', clause) |
dmsg("words:", words, min_level=2) |
|
for index, word in enumerate(words): |
sentence = sentence.replace(word, str(translated_words[index])) |
print(sentence) |
offset = 0 |
while offset < len(words): |
translation = None |
|
# replace punctuation |
for symbol, replacement in ({" - ": ", "}).items(): |
sentence = sentence.replace(symbol, replacement) |
for i in reversed(range(offset + 1, len(words) + 1)): |
phrase = ' '.join(words[offset:i]) |
|
print(sentence) |
dmsg("phrase:", phrase, min_level=2) |
|
translation = translate(phrase) |
|
if translation is not None: |
dmsg("phrase-translation:", translation, min_level=2) |
dmsg("words[{0}:{1}] = [\"{2}\"]".format(offset, i, translation), min_level=2) |
words[offset:i] = [translation] |
offset += i - 1 |
break |
|
if translation is None: |
dmsg("phrase-translation:", translation, min_level=2) |
offset += 1 |
|
dmsg("words-translation:", words, min_level=2) |