WebSVN - LCARS - Diff - Rev 293 and 294 - /trunk/tools/eazytrans/vuh.py


#!/usr/bin/env python3

'''
Created on 2014-10-20

@author: Thomas 'PointedEars' Lahn <mail@PointedEars.de>
'''
from sys import argv, stderr
from re import findall, DOTALL, match, sub, compile, \
    escape, search
from os.path import basename
from functools import cmp_to_key
from Dictionary import Dictionary, dmsg, \
    sort_dict_alnum_english_key

dictionary = {}

prepositions = {
    "fi'": 'on',
    "na'": 'at|to',
    "t'": 'of'
}

def cli_help():
    print('Usage: {0} TEXT...'.format(basename(argv[0])))

def get_sort_dict_alnum_vulcan_key():
    letters = list(map(str.lower, [
        " ", 'S', 'T', 'P', 'K', 'R', 'L', 'A', 'Sh', 'O', 'U', 'D',
        'V', 'Kh', 'E', 'H', 'G', 'Ch', 'I', 'N', 'Zh', 'M', 'Y', 'F', 'Z',
        'Th', 'W', 'B', "'", '-']))
    letter_values = dict(map(lambda x: (x[1], x[0]), enumerate(letters)))
    letters_re = compile(r'(?:{0})'.format('|'.join(sorted(letters, key=lambda char:-len(char)))))

    def sort_dict_alnum_vulcan (a, b):
        # split into Vulcan letters
        a = findall(letters_re, sort_dict_alnum_english_key(a))
        b = findall(letters_re, sort_dict_alnum_english_key(b))

        if len(a) < len(b):
            for index, char in enumerate(a):
                diff = letter_values[char] - letter_values[b[index]]
                if diff != 0:
                    return diff
            return -1

        # len(b) <= len(a)
        for index, char in enumerate(b):
            diff = letter_values[a[index]] - letter_values[char]
            if diff != 0:
                return diff

        return 1 if len(b) < len(a) else 0

    return cmp_to_key(sort_dict_alnum_vulcan)

class VulcanDictionary(Dictionary):
    def translate (self, phrase, search_prefix=True, search_plural=True):
        dictionary = self

        translation = dictionary.get(phrase.lower(), None)
        if translation is not None:
            translation['vuh'] = phrase
            return translation
        else:
            if search_prefix:
                # find prefix
                for preposition in prepositions:
                    prefix = match(escape(preposition), phrase)
                    if prefix is not None:
                        prefix_translation = self.translate(prefix.group(0))
                        if prefix_translation is not None:
                            tail = sub(preposition, '', phrase)
                            tail_translation = self.translate(tail, search_prefix=False)
                            if tail_translation is not None:
                                return [prefix_translation, tail_translation]
            elif search_plural:
                # find plural
                suffix = search(r'lar$', phrase)
                if suffix is not None:
                    head = sub(r'lar$', '', phrase)
                    head_translation = self.translate(head, search_prefix=False, search_plural=False)
                    if head_translation is not None:
                        head_translation = dict(head_translation)
                        head_translation['en'] += ' (pl.)'
                        return head_translation

        return None

if __name__ == '__main__':
    if len(argv) < 2:
        print('Nothing to translate.', end='\n\n', file=stderr)
        cli_help()
        exit(1)

    text = argv[1]

    dictionary = VulcanDictionary(dictionary)
    dictionary.load('vuh-gol-en.dict.zdb.txt', 'vuh')
    dictionary.clean()

#     try:
#         for phrase, data in OrderedDict(sorted(
#             dictionary.items(),
#             key=get_sort_dict_alnum_vulcan_key()
#         )).items():
#             print(phrase, "=", data)
#     except BrokenPipeError:
#         pass

    dmsg("text:", text, min_level=2)
    sentences = findall(r'(?!\s+)(?:.+?\.{1,3}|.+$)', text, DOTALL)
    dmsg("sentences:", sentences, min_level=2)
    for sentence in sentences:
        dmsg("sentence:", sentence, min_level=2)

        clauses = findall(r'(?!\s+)(?:.+?(?:\s+-\s*|\s*[–—]\s*|\.{1,3}|.+$))', sentence, DOTALL)
        dmsg("clauses:", clauses, min_level=2)
        for clause in clauses:
            dmsg("clause:", clause, min_level=2)

            words = findall(r'[^\s.]+', clause)
            dmsg("words:", words, min_level=2)

            offset = 0
            while offset < len(words):
                translation = None

                for i in range(len(words), offset, -1):
                    dmsg("words[{0}:{1}] = {2}".format(offset, i, words[offset:i]), min_level=2)
                    phrase = ' '.join(words[offset:i])

                    dmsg("phrase:", phrase, min_level=2)

                    translation = dictionary.translate(phrase)

                    if translation is not None:
                        dmsg("phrase-translation:", translation, min_level=2)
                        dmsg("words[{0}:{1}] = [\"{2}\"]".format(offset, i, translation), min_level=2)
                        words[offset:i] = [translation]
                        offset += i - offset
                        break

                if translation is None:
                    dmsg("phrase-translation:", translation, min_level=2)
                    offset += 1

            dmsg("words-translation:", words, min_level=2)
 

Rev 293	Rev 294
1	#!/usr/bin/env python3	1	#!/usr/bin/env python3
2		2
3	'''	3	'''
4	Created on 2014-10-20	4	Created on 2014-10-20
5		5
6	@author: Thomas 'PointedEars' Lahn <mail@PointedEars.de>	6	@author: Thomas 'PointedEars' Lahn <mail@PointedEars.de>
7	'''	7	'''
8	from sys import argv, stderr	8	from sys import argv, stderr
9	from re import findall, DOTALL, match, sub, compile, \	9	from re import findall, DOTALL, match, sub, compile, \
10	escape, search	10	escape, search
11	from os.path import basename	11	from os.path import basename
12	from functools import cmp_to_key	12	from functools import cmp_to_key
13	from Dictionary import Dictionary, dmsg, \	13	from Dictionary import Dictionary, dmsg, \
14	sort_dict_alnum_english_key	14	sort_dict_alnum_english_key
15		15
16	dictionary = {}	16	dictionary = {}
17		17
18	prepositions = {	18	prepositions = {
19	"fi'": 'on',	19	"fi'": 'on',
20	"na'": 'at\|to',	20	"na'": 'at\|to',
21	"t'": 'of'	21	"t'": 'of'
22	}	22	}
23		23
24	def cli_help():	24	def cli_help():
25	print('Usage: {0} TEXT...'.format(basename(argv[0])))	25	print('Usage: {0} TEXT...'.format(basename(argv[0])))
26		26
27	def get_sort_dict_alnum_vulcan_key():	27	def get_sort_dict_alnum_vulcan_key():
28	letters = list(map(str.lower, [	28	letters = list(map(str.lower, [
29	" ", 'S', 'T', 'P', 'K', 'R', 'L', 'A', 'Sh', 'O', 'U', 'D',	29	" ", 'S', 'T', 'P', 'K', 'R', 'L', 'A', 'Sh', 'O', 'U', 'D',
30	'V', 'Kh', 'E', 'H', 'G', 'Ch', 'I', 'N', 'Zh', 'M', 'Y', 'F', 'Z',	30	'V', 'Kh', 'E', 'H', 'G', 'Ch', 'I', 'N', 'Zh', 'M', 'Y', 'F', 'Z',
31	'Th', 'W', 'B', "'", '-']))	31	'Th', 'W', 'B', "'", '-']))
32	letter_values = dict(map(lambda x: (x[1], x[0]), enumerate(letters)))	32	letter_values = dict(map(lambda x: (x[1], x[0]), enumerate(letters)))
33	letters_re = compile(r'(?:{0})'.format('\|'.join(sorted(letters, key=lambda char:-len(char)))))	33	letters_re = compile(r'(?:{0})'.format('\|'.join(sorted(letters, key=lambda char:-len(char)))))
34		34
35	def sort_dict_alnum_vulcan (a, b):	35	def sort_dict_alnum_vulcan (a, b):
36	# split into Vulcan letters	36	# split into Vulcan letters
37	a = findall(letters_re, sort_dict_alnum_english_key(a))	37	a = findall(letters_re, sort_dict_alnum_english_key(a))
38	b = findall(letters_re, sort_dict_alnum_english_key(b))	38	b = findall(letters_re, sort_dict_alnum_english_key(b))
39		39
40	if len(a) < len(b):	40	if len(a) < len(b):
41	for index, char in enumerate(a):	41	for index, char in enumerate(a):
42	diff = letter_values[char] - letter_values[b[index]]	42	diff = letter_values[char] - letter_values[b[index]]
43	if diff != 0:	43	if diff != 0:
44	return diff	44	return diff
45	return -1	45	return -1
46		46
47	# len(b) <= len(a)	47	# len(b) <= len(a)
48	for index, char in enumerate(b):	48	for index, char in enumerate(b):
49	diff = letter_values[a[index]] - letter_values[char]	49	diff = letter_values[a[index]] - letter_values[char]
50	if diff != 0:	50	if diff != 0:
51	return diff	51	return diff
52		52
53	return 1 if len(b) < len(a) else 0	53	return 1 if len(b) < len(a) else 0
54		54
55	return cmp_to_key(sort_dict_alnum_vulcan)	55	return cmp_to_key(sort_dict_alnum_vulcan)
56		56
57	class VulcanDictionary(Dictionary):	57	class VulcanDictionary(Dictionary):
58	def translate (self, phrase, search_prefix=True, search_plural=True):	58	def translate (self, phrase, search_prefix=True, search_plural=True):
59	dictionary = self	59	dictionary = self
60		60
61	translation = dictionary.get(phrase.lower(), None)	61	translation = dictionary.get(phrase.lower(), None)
62	if translation is not None:	62	if translation is not None:
63	translation['vuh'] = phrase	63	translation['vuh'] = phrase
64	return translation	64	return translation
65	else:	65	else:
66	if search_prefix:	66	if search_prefix:
67	# find prefix	67	# find prefix
68	for preposition in prepositions:	68	for preposition in prepositions:
69	prefix = match(escape(preposition), phrase)	69	prefix = match(escape(preposition), phrase)
70	if prefix is not None:	70	if prefix is not None:
71	prefix_translation = self.translate(prefix.group(0))	71	prefix_translation = self.translate(prefix.group(0))
72	if prefix_translation is not None:	72	if prefix_translation is not None:
73	tail = sub(preposition, '', phrase)	73	tail = sub(preposition, '', phrase)
74	tail_translation = self.translate(tail, search_prefix=False)	74	tail_translation = self.translate(tail, search_prefix=False)
75	if tail_translation is not None:	75	if tail_translation is not None:
76	return [prefix_translation, tail_translation]	76	return [prefix_translation, tail_translation]
77	elif search_plural:	77	elif search_plural:
78	# find plural	78	# find plural
79	suffix = search(r'lar$', phrase)	79	suffix = search(r'lar$', phrase)
80	if suffix is not None:	80	if suffix is not None:
81	head = sub(r'lar$', '', phrase)	81	head = sub(r'lar$', '', phrase)
82	head_translation = self.translate(head, search_prefix=False, search_plural=False)	82	head_translation = self.translate(head, search_prefix=False, search_plural=False)
83	if head_translation is not None:	83	if head_translation is not None:
84	head_translation = dict(head_translation)	84	head_translation = dict(head_translation)
85	head_translation['en'] += ' (pl.)'	85	head_translation['en'] += ' (pl.)'
86	return head_translation	86	return head_translation
87		87
88	return None	88	return None
89		89
90	if __name__ == '__main__':	90	if __name__ == '__main__':
91	if len(argv) < 2:	91	if len(argv) < 2:
92	print('Nothing to translate.', end='\n\n', file=stderr)	92	print('Nothing to translate.', end='\n\n', file=stderr)
93	cli_help()	93	cli_help()
94	exit(1)	94	exit(1)
95		95
96	text = argv[1]	96	text = argv[1]
97		97
98	dictionary = VulcanDictionary(dictionary)	98	dictionary = VulcanDictionary(dictionary)
99	dictionary.load('vuh-gol-en.dict.zdb.txt')	99	dictionary.load('vuh-gol-en.dict.zdb.txt', 'vuh')
100	dictionary.clean()	100	dictionary.clean()
101		101
102	# try:	102	# try:
103	# for phrase, data in OrderedDict(sorted(	103	# for phrase, data in OrderedDict(sorted(
104	# dictionary.items(),	104	# dictionary.items(),
105	# key=get_sort_dict_alnum_vulcan_key()	105	# key=get_sort_dict_alnum_vulcan_key()
106	# )).items():	106	# )).items():
107	# print(phrase, "=", data)	107	# print(phrase, "=", data)
108	# except BrokenPipeError:	108	# except BrokenPipeError:
109	# pass	109	# pass
110		110
111	dmsg("text:", text, min_level=2)	111	dmsg("text:", text, min_level=2)
112	sentences = findall(r'(?!\s+)(?:.+?\.{1,3}\|.+$)', text, DOTALL)	112	sentences = findall(r'(?!\s+)(?:.+?\.{1,3}\|.+$)', text, DOTALL)
113	dmsg("sentences:", sentences, min_level=2)	113	dmsg("sentences:", sentences, min_level=2)
114	for sentence in sentences:	114	for sentence in sentences:
115	dmsg("sentence:", sentence, min_level=2)	115	dmsg("sentence:", sentence, min_level=2)
116		116
117	clauses = findall(r'(?!\s+)(?:.+?(?:\s+-\s\|\s[–—]\s*\|\.{1,3}\|.+$))', sentence, DOTALL)	117	clauses = findall(r'(?!\s+)(?:.+?(?:\s+-\s\|\s[–—]\s*\|\.{1,3}\|.+$))', sentence, DOTALL)
118	dmsg("clauses:", clauses, min_level=2)	118	dmsg("clauses:", clauses, min_level=2)
119	for clause in clauses:	119	for clause in clauses:
120	dmsg("clause:", clause, min_level=2)	120	dmsg("clause:", clause, min_level=2)
121		121
122	words = findall(r'[^\s.]+', clause)	122	words = findall(r'[^\s.]+', clause)
123	dmsg("words:", words, min_level=2)	123	dmsg("words:", words, min_level=2)
124		124
125	offset = 0	125	offset = 0
126	while offset < len(words):	126	while offset < len(words):
127	translation = None	127	translation = None
128		128
129	for i in range(len(words), offset, -1):	129	for i in range(len(words), offset, -1):
130	dmsg("words[{0}:{1}] = {2}".format(offset, i, words[offset:i]), min_level=2)	130	dmsg("words[{0}:{1}] = {2}".format(offset, i, words[offset:i]), min_level=2)
131	phrase = ' '.join(words[offset:i])	131	phrase = ' '.join(words[offset:i])
132		132
133	dmsg("phrase:", phrase, min_level=2)	133	dmsg("phrase:", phrase, min_level=2)
134		134
135	translation = dictionary.translate(phrase)	135	translation = dictionary.translate(phrase)
136		136
137	if translation is not None:	137	if translation is not None:
138	dmsg("phrase-translation:", translation, min_level=2)	138	dmsg("phrase-translation:", translation, min_level=2)
139	dmsg("words[{0}:{1}] = [\"{2}\"]".format(offset, i, translation), min_level=2)	139	dmsg("words[{0}:{1}] = [\"{2}\"]".format(offset, i, translation), min_level=2)
140	words[offset:i] = [translation]	140	words[offset:i] = [translation]
141	offset += i - offset	141	offset += i - offset
142	break	142	break
143		143
144	if translation is None:	144	if translation is None:
145	dmsg("phrase-translation:", translation, min_level=2)	145	dmsg("phrase-translation:", translation, min_level=2)
146	offset += 1	146	offset += 1
147		147
148	dmsg("words-translation:", words, min_level=2)	148	dmsg("words-translation:", words, min_level=2)
149		149

Subversion Repositories LCARS

(root)/trunk/tools/eazytrans/vuh.py - Rev 293 → 294