WebSVN - LCARS - Diff - Rev 294 and 296 - /trunk/tools/eazytrans/vuh.py


#!/usr/bin/env python3

'''
Created on 2014-10-20

@author: Thomas 'PointedEars' Lahn <mail@PointedEars.de>
'''
from sys import argv, stderr
from re import findall, DOTALL, match, sub, compile, \
    escape, search
from os.path import basename
from functools import cmp_to_key
from Dictionary import Dictionary, dmsg, \
    sort_dict_alnum_english_key

dictionary = {}

prepositions = {
    "fi'": 'on',
    "na'": 'at|to',
    "t'": 'of'
}

def cli_help():
    print('Usage: {0} TEXT...'.format(basename(argv[0])))

def get_sort_dict_alnum_vulcan_key ():
    letters = list(map(str.lower, [
        " ", 'S', 'T', 'P', 'K', 'R', 'L', 'A', 'Sh', 'O', 'U', 'D',
        'V', 'Kh', 'E', 'H', 'G', 'Ch', 'I', 'N', 'Zh', 'M', 'Y', 'F', 'Z',
        'Th', 'W', 'B', "'", '-']))
    letter_values = dict(map(lambda x: (x[1], x[0]), enumerate(letters)))
    letters_re = compile(r'(?:{0})'.format('|'.join(sorted(letters, key=lambda char:-len(char)))))

    def sort_dict_alnum_vulcan (a, b):
        # split into Vulcan letters
        a = findall(letters_re, sort_dict_alnum_english_key(a))
        b = findall(letters_re, sort_dict_alnum_english_key(b))

        if len(a) < len(b):
            for index, char in enumerate(a):
                diff = letter_values[char] - letter_values[b[index]]
                if diff != 0:
                    return diff
            return -1

        # len(b) <= len(a)
        for index, char in enumerate(b):
            diff = letter_values[a[index]] - letter_values[char]
            if diff != 0:
                return diff

        return 1 if len(b) < len(a) else 0

    return cmp_to_key(sort_dict_alnum_vulcan)

class VulcanDictionary (Dictionary):
    def translate (self, phrase, search_prefix=True, search_plural=True):
        dictionary = self

        translation = super().translate(phrase)
        if translation is not None:
 
            return translation
        else:
            expr_translation = dictionary.translate_expression(phrase)
            if expr_translation is not None:
                return expr_translation

            if search_prefix:
                # find prefix
                for preposition in prepositions:
                    prefix = match(escape(preposition), phrase)
                    if prefix is not None:
                        prefix_translation = self.translate(prefix.group(0))
                        if prefix_translation is not None:
                            tail = sub(preposition, '', phrase)
                            tail_translation = self.translate(tail, search_prefix=False)
                            if tail_translation is not None:
                                return [prefix_translation, tail_translation]
            elif search_plural:
                # find plural
                suffix = search(r'lar$', phrase)
                if suffix is not None:
                    head = sub(r'lar$', '', phrase)
                    head_translation = self.translate(head, search_prefix=False, search_plural=False)
                    if head_translation is not None:
                        head_translation = dict(head_translation)
                        head_translation['en'] += ' (pl.)'
                        return head_translation

        return None

if __name__ == '__main__':
    if len(argv) < 2:
        print('Nothing to translate.', end='\n\n', file=stderr)
        cli_help()
        exit(1)

    text = ' '.join(argv[1:])

    dictionary = VulcanDictionary(dictionary)
    dictionary.load('vuh-gol-en.dict.zdb.txt', 'vuh')
    dictionary.clean()

#     try:
#         for phrase, data in OrderedDict(sorted(
#             dictionary.items(),
#             key=get_sort_dict_alnum_vulcan_key()
#         )).items():
#             print(phrase, "=", data)
#     except BrokenPipeError:
#         pass

    dmsg("text:", text, min_level=2)
    sentences = findall(r'(?!\s+)(?:.+?\.{1,3}|.+$)', text, DOTALL)
    dmsg("sentences:", sentences, min_level=2)
    for sentence in sentences:
        dmsg("sentence:", sentence, min_level=2)

        clauses = findall(r'(?!\s+)(?:.+?(?:\s+-\s*|\s*[–—]\s*|\.{1,3}|.+$))', sentence, DOTALL)
        dmsg("clauses:", clauses, min_level=2)
        for clause in clauses:
            dmsg("clause:", clause, min_level=2)

            words = findall(r'[^\s.]+', clause)
            dmsg("words:", words, min_level=2)

            offset = 0
            while offset < len(words):
                translation = None

                for i in range(len(words), offset, -1):
                    dmsg("words[{0}:{1}] = {2}".format(offset, i, words[offset:i]), min_level=2)
                    phrase = ' '.join(words[offset:i])

                    dmsg("phrase:", phrase, min_level=2)

                    translation = dictionary.translate(phrase)

                    if translation is not None:
                        dmsg("phrase-translation:", translation, min_level=2)
                        dmsg("words[{0}:{1}] = [\"{2}\"]".format(offset, i, translation), min_level=2)
                        words[offset:i] = [translation]
                        offset += i - offset
                        break

                if translation is None:
                    dmsg("phrase-translation:", translation, min_level=2)
                    offset += 1

            dmsg("words-translation:", words, min_level=2)
            dmsg("words-translation-reduced:",
                list(map(
                    lambda word:
                        word['en']
                        if (hasattr(word, "get") and word.get('en', None) is not None)
                        else word,
                    words)),
                min_level=2)
            # dmsg(dictionary._expressions)
 

Rev 294	Rev 296
1	#!/usr/bin/env python3	1	#!/usr/bin/env python3
2		2
3	'''	3	'''
4	Created on 2014-10-20	4	Created on 2014-10-20
5		5
6	@author: Thomas 'PointedEars' Lahn <mail@PointedEars.de>	6	@author: Thomas 'PointedEars' Lahn <mail@PointedEars.de>
7	'''	7	'''
8	from sys import argv, stderr	8	from sys import argv, stderr
9	from re import findall, DOTALL, match, sub, compile, \	9	from re import findall, DOTALL, match, sub, compile, \
10	escape, search	10	escape, search
11	from os.path import basename	11	from os.path import basename
12	from functools import cmp_to_key	12	from functools import cmp_to_key
13	from Dictionary import Dictionary, dmsg, \	13	from Dictionary import Dictionary, dmsg, \
14	sort_dict_alnum_english_key	14	sort_dict_alnum_english_key
15		15
16	dictionary = {}	16	dictionary = {}
17		17
18	prepositions = {	18	prepositions = {
19	"fi'": 'on',	19	"fi'": 'on',
20	"na'": 'at\|to',	20	"na'": 'at\|to',
21	"t'": 'of'	21	"t'": 'of'
22	}	22	}
23		23
24	def cli_help():	24	def cli_help():
25	print('Usage: {0} TEXT...'.format(basename(argv[0])))	25	print('Usage: {0} TEXT...'.format(basename(argv[0])))
26		26
27	def get_sort_dict_alnum_vulcan_key():	27	def get_sort_dict_alnum_vulcan_key ():
28	letters = list(map(str.lower, [	28	letters = list(map(str.lower, [
29	" ", 'S', 'T', 'P', 'K', 'R', 'L', 'A', 'Sh', 'O', 'U', 'D',	29	" ", 'S', 'T', 'P', 'K', 'R', 'L', 'A', 'Sh', 'O', 'U', 'D',
30	'V', 'Kh', 'E', 'H', 'G', 'Ch', 'I', 'N', 'Zh', 'M', 'Y', 'F', 'Z',	30	'V', 'Kh', 'E', 'H', 'G', 'Ch', 'I', 'N', 'Zh', 'M', 'Y', 'F', 'Z',
31	'Th', 'W', 'B', "'", '-']))	31	'Th', 'W', 'B', "'", '-']))
32	letter_values = dict(map(lambda x: (x[1], x[0]), enumerate(letters)))	32	letter_values = dict(map(lambda x: (x[1], x[0]), enumerate(letters)))
33	letters_re = compile(r'(?:{0})'.format('\|'.join(sorted(letters, key=lambda char:-len(char)))))	33	letters_re = compile(r'(?:{0})'.format('\|'.join(sorted(letters, key=lambda char:-len(char)))))
34		34
35	def sort_dict_alnum_vulcan (a, b):	35	def sort_dict_alnum_vulcan (a, b):
36	# split into Vulcan letters	36	# split into Vulcan letters
37	a = findall(letters_re, sort_dict_alnum_english_key(a))	37	a = findall(letters_re, sort_dict_alnum_english_key(a))
38	b = findall(letters_re, sort_dict_alnum_english_key(b))	38	b = findall(letters_re, sort_dict_alnum_english_key(b))
39		39
40	if len(a) < len(b):	40	if len(a) < len(b):
41	for index, char in enumerate(a):	41	for index, char in enumerate(a):
42	diff = letter_values[char] - letter_values[b[index]]	42	diff = letter_values[char] - letter_values[b[index]]
43	if diff != 0:	43	if diff != 0:
44	return diff	44	return diff
45	return -1	45	return -1
46		46
47	# len(b) <= len(a)	47	# len(b) <= len(a)
48	for index, char in enumerate(b):	48	for index, char in enumerate(b):
49	diff = letter_values[a[index]] - letter_values[char]	49	diff = letter_values[a[index]] - letter_values[char]
50	if diff != 0:	50	if diff != 0:
51	return diff	51	return diff
52		52
53	return 1 if len(b) < len(a) else 0	53	return 1 if len(b) < len(a) else 0
54		54
55	return cmp_to_key(sort_dict_alnum_vulcan)	55	return cmp_to_key(sort_dict_alnum_vulcan)
56		56
57	class VulcanDictionary(Dictionary):	57	class VulcanDictionary (Dictionary):
58	def translate (self, phrase, search_prefix=True, search_plural=True):	58	def translate (self, phrase, search_prefix=True, search_plural=True):
59	dictionary = self	59	dictionary = self
60		60
61	translation = dictionary.get(phrase.lower(), None)	61	translation = super().translate(phrase)
62	if translation is not None:	62	if translation is not None:
63	translation['vuh'] = phrase	-
64	return translation	63	return translation
65	else:	64	else:
-		65	expr_translation = dictionary.translate_expression(phrase)
-		66	if expr_translation is not None:
-		67	return expr_translation
-		68
66	if search_prefix:	69	if search_prefix:
67	# find prefix	70	# find prefix
68	for preposition in prepositions:	71	for preposition in prepositions:
69	prefix = match(escape(preposition), phrase)	72	prefix = match(escape(preposition), phrase)
70	if prefix is not None:	73	if prefix is not None:
71	prefix_translation = self.translate(prefix.group(0))	74	prefix_translation = self.translate(prefix.group(0))
72	if prefix_translation is not None:	75	if prefix_translation is not None:
73	tail = sub(preposition, '', phrase)	76	tail = sub(preposition, '', phrase)
74	tail_translation = self.translate(tail, search_prefix=False)	77	tail_translation = self.translate(tail, search_prefix=False)
75	if tail_translation is not None:	78	if tail_translation is not None:
76	return [prefix_translation, tail_translation]	79	return [prefix_translation, tail_translation]
77	elif search_plural:	80	elif search_plural:
78	# find plural	81	# find plural
79	suffix = search(r'lar$', phrase)	82	suffix = search(r'lar$', phrase)
80	if suffix is not None:	83	if suffix is not None:
81	head = sub(r'lar$', '', phrase)	84	head = sub(r'lar$', '', phrase)
82	head_translation = self.translate(head, search_prefix=False, search_plural=False)	85	head_translation = self.translate(head, search_prefix=False, search_plural=False)
83	if head_translation is not None:	86	if head_translation is not None:
84	head_translation = dict(head_translation)	87	head_translation = dict(head_translation)
85	head_translation['en'] += ' (pl.)'	88	head_translation['en'] += ' (pl.)'
86	return head_translation	89	return head_translation
87		90
88	return None	91	return None
89		92
90	if __name__ == '__main__':	93	if __name__ == '__main__':
91	if len(argv) < 2:	94	if len(argv) < 2:
92	print('Nothing to translate.', end='\n\n', file=stderr)	95	print('Nothing to translate.', end='\n\n', file=stderr)
93	cli_help()	96	cli_help()
94	exit(1)	97	exit(1)
95		98
96	text = argv[1]	99	text = ' '.join(argv[1:])
97		100
98	dictionary = VulcanDictionary(dictionary)	101	dictionary = VulcanDictionary(dictionary)
99	dictionary.load('vuh-gol-en.dict.zdb.txt', 'vuh')	102	dictionary.load('vuh-gol-en.dict.zdb.txt', 'vuh')
100	dictionary.clean()	103	dictionary.clean()
101		104
102	# try:	105	# try:
103	# for phrase, data in OrderedDict(sorted(	106	# for phrase, data in OrderedDict(sorted(
104	# dictionary.items(),	107	# dictionary.items(),
105	# key=get_sort_dict_alnum_vulcan_key()	108	# key=get_sort_dict_alnum_vulcan_key()
106	# )).items():	109	# )).items():
107	# print(phrase, "=", data)	110	# print(phrase, "=", data)
108	# except BrokenPipeError:	111	# except BrokenPipeError:
109	# pass	112	# pass
110		113
111	dmsg("text:", text, min_level=2)	114	dmsg("text:", text, min_level=2)
112	sentences = findall(r'(?!\s+)(?:.+?\.{1,3}\|.+$)', text, DOTALL)	115	sentences = findall(r'(?!\s+)(?:.+?\.{1,3}\|.+$)', text, DOTALL)
113	dmsg("sentences:", sentences, min_level=2)	116	dmsg("sentences:", sentences, min_level=2)
114	for sentence in sentences:	117	for sentence in sentences:
115	dmsg("sentence:", sentence, min_level=2)	118	dmsg("sentence:", sentence, min_level=2)
116		119
117	clauses = findall(r'(?!\s+)(?:.+?(?:\s+-\s\|\s[–—]\s*\|\.{1,3}\|.+$))', sentence, DOTALL)	120	clauses = findall(r'(?!\s+)(?:.+?(?:\s+-\s\|\s[–—]\s*\|\.{1,3}\|.+$))', sentence, DOTALL)
118	dmsg("clauses:", clauses, min_level=2)	121	dmsg("clauses:", clauses, min_level=2)
119	for clause in clauses:	122	for clause in clauses:
120	dmsg("clause:", clause, min_level=2)	123	dmsg("clause:", clause, min_level=2)
121		124
122	words = findall(r'[^\s.]+', clause)	125	words = findall(r'[^\s.]+', clause)
123	dmsg("words:", words, min_level=2)	126	dmsg("words:", words, min_level=2)
124		127
125	offset = 0	128	offset = 0
126	while offset < len(words):	129	while offset < len(words):
127	translation = None	130	translation = None
128		131
129	for i in range(len(words), offset, -1):	132	for i in range(len(words), offset, -1):
130	dmsg("words[{0}:{1}] = {2}".format(offset, i, words[offset:i]), min_level=2)	133	dmsg("words[{0}:{1}] = {2}".format(offset, i, words[offset:i]), min_level=2)
131	phrase = ' '.join(words[offset:i])	134	phrase = ' '.join(words[offset:i])
132		135
133	dmsg("phrase:", phrase, min_level=2)	136	dmsg("phrase:", phrase, min_level=2)
134		137
135	translation = dictionary.translate(phrase)	138	translation = dictionary.translate(phrase)
136		139
137	if translation is not None:	140	if translation is not None:
138	dmsg("phrase-translation:", translation, min_level=2)	141	dmsg("phrase-translation:", translation, min_level=2)
139	dmsg("words[{0}:{1}] = [\"{2}\"]".format(offset, i, translation), min_level=2)	142	dmsg("words[{0}:{1}] = [\"{2}\"]".format(offset, i, translation), min_level=2)
140	words[offset:i] = [translation]	143	words[offset:i] = [translation]
141	offset += i - offset	144	offset += i - offset
142	break	145	break
143		146
144	if translation is None:	147	if translation is None:
145	dmsg("phrase-translation:", translation, min_level=2)	148	dmsg("phrase-translation:", translation, min_level=2)
146	offset += 1	149	offset += 1
147		150
148	dmsg("words-translation:", words, min_level=2)	151	dmsg("words-translation:", words, min_level=2)
-		152	dmsg("words-translation-reduced:",
-		153	list(map(
-		154	lambda word:
-		155	word['en']
-		156	if (hasattr(word, "get") and word.get('en', None) is not None)
-		157	else word,
-		158	words)),
-		159	min_level=2)
-		160	# dmsg(dictionary._expressions)
149		161

Subversion Repositories LCARS

(root)/trunk/tools/eazytrans/vuh.py - Rev 294 → 296