gluon.languages

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 This file is part of the web2py Web Framework 6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 8 9 Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine) 10 <dbdevelop@gmail.com> 11 """ 12 13 import os 14 import re 15 import sys 16 import pkgutil 17 import logging 18 from cgi import escape 19 from threading import RLock 20 21 try: 22 import copyreg as copy_reg # python 3 23 except ImportError: 24 import copy_reg # python 2 25 26 from gluon.portalocker import read_locked, LockedFile 27 from utf8 import Utf8 28 29 from gluon.fileutils import listdir 30 import gluon.settings as settings 31 from gluon.cfs import getcfs 32 from gluon.html import XML, xmlescape 33 from gluon.contrib.markmin.markmin2html import render, markmin_escape 34 from string import maketrans 35 36 __all__ = ['translator', 'findT', 'update_all_languages'] 37 38 ostat = os.stat 39 oslistdir = os.listdir 40 pjoin = os.path.join 41 pexists = os.path.exists 42 pdirname = os.path.dirname 43 isdir = os.path.isdir 44 is_gae = False # settings.global_settings.web2py_runtime_gae 45 46 DEFAULT_LANGUAGE = 'en' 47 DEFAULT_LANGUAGE_NAME = 'English' 48 49 # DEFAULT PLURAL-FORMS RULES: 50 # language doesn't use plural forms 51 DEFAULT_NPLURALS = 1 52 # only one singular/plural form is used 53 DEFAULT_GET_PLURAL_ID = lambda n: 0 54 # word is unchangeable 55 DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word 56 57 NUMBERS = (int, long, float) 58 59 # pattern to find T(blah blah blah) expressions 60 PY_STRING_LITERAL_RE = r'(?<=[^\w]T$)(?P<name>'\ 61 + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\ 62 + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\ 63 + r'(?:"(?:[^"\\]|\\.)*"))' 64 65 regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL) 66 regex_param = re.compile(r'{(?P<s>.+?)}') 67 68 # pattern for a valid accept_language 69 regex_language = \ 70 re.compile('([a-z]{2,3}(?:\-[a-z]{2})?(?:\-[a-z]{2})?)(?:[,;]|$)') 71 regex_langfile = re.compile('^[a-z]{2,3}(-[a-z]{2})?\.py$') 72 regex_backslash = re.compile(r"\\([\\{}%])") 73 regex_plural = re.compile('%({.+?})') 74 regex_plural_dict = re.compile('^{(?P<w>[^()[\]][^()[\]]*?)\((?P<n>[^()\[\]]+)$}$') # %%{word(varname or number)} 75 regex_plural_tuple = re.compile( 76 '^{(?P<w>[^[\]()]+)(?:\[(?P<i>\d+)\])?}$') # %%{word[index]} or %%{word} 77 regex_plural_file = re.compile('^plural-[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$') 78 79

80 -def safe_eval(text):

81 if text.strip(): 82 try: 83 import ast 84 return ast.literal_eval(text) 85 except ImportError: 86 return eval(text, {}, {}) 87 return None

88 89 # used as default filter in translator.M() 90 91

92 -def markmin(s):

93 def markmin_aux(m): 94 return '{%s}' % markmin_escape(m.group('s'))

95 return render(regex_param.sub(markmin_aux, s), 96 sep='br', autolinks=None, id_prefix='') 97 98 # UTF8 helper functions 99 100

101 -def upper_fun(s):

102 return unicode(s, 'utf-8').upper().encode('utf-8')

103 104

105 -def title_fun(s):

106 return unicode(s, 'utf-8').title().encode('utf-8')

107 108

109 -def cap_fun(s):

110 return unicode(s, 'utf-8').capitalize().encode('utf-8')

111 ttab_in = maketrans("\\%{}", '\x1c\x1d\x1e\x1f') 112 ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}") 113 114 # cache of translated messages: 115 # global_language_cache: 116 # { 'languages/xx.py': 117 # ( {"def-message": "xx-message", 118 # ... 119 # "def-message": "xx-message"}, lock_object ) 120 # 'languages/yy.py': ( {dict}, lock_object ) 121 # ... 122 # } 123 124 global_language_cache = {} 125 126

127 -def get_from_cache(cache, val, fun):

128 lang_dict, lock = cache 129 lock.acquire() 130 try: 131 result = lang_dict.get(val) 132 finally: 133 lock.release() 134 if result: 135 return result 136 lock.acquire() 137 try: 138 result = lang_dict.setdefault(val, fun()) 139 finally: 140 lock.release() 141 return result

142 143

144 -def clear_cache(filename):

145 cache = global_language_cache.setdefault( 146 filename, ({}, RLock())) 147 lang_dict, lock = cache 148 lock.acquire() 149 try: 150 lang_dict.clear() 151 finally: 152 lock.release()

153 154

155 -def read_dict_aux(filename):

156 lang_text = read_locked(filename).replace('\r\n', '\n') 157 clear_cache(filename) 158 try: 159 return safe_eval(lang_text) or {} 160 except Exception: 161 e = sys.exc_info()[1] 162 status = 'Syntax error in %s (%s)' % (filename, e) 163 logging.error(status) 164 return {'__corrupted__': status}

165 166

167 -def read_dict(filename):

168 """ return dictionary with translation messages 169 """ 170 return getcfs('lang:' + filename, filename, 171 lambda: read_dict_aux(filename))

172 173

174 -def read_possible_plural_rules():

175 """ 176 create list of all possible plural rules files 177 result is cached in PLURAL_RULES dictionary to increase speed 178 """ 179 plurals = {} 180 try: 181 import gluon.contrib.plural_rules as package 182 for importer, modname, ispkg in pkgutil.iter_modules(package.__path__): 183 if len(modname) == 2: 184 module = __import__(package.__name__ + '.' + modname, 185 fromlist=[modname]) 186 lang = modname 187 pname = modname + '.py' 188 nplurals = getattr(module, 'nplurals', DEFAULT_NPLURALS) 189 get_plural_id = getattr( 190 module, 'get_plural_id', 191 DEFAULT_GET_PLURAL_ID) 192 construct_plural_form = getattr( 193 module, 'construct_plural_form', 194 DEFAULT_CONSTRUCT_PLURAL_FORM) 195 plurals[lang] = (lang, nplurals, get_plural_id, 196 construct_plural_form) 197 except ImportError: 198 e = sys.exc_info()[1] 199 logging.warn('Unable to import plural rules: %s' % e) 200 return plurals

201 202 PLURAL_RULES = read_possible_plural_rules() 203 204

205 -def read_possible_languages_aux(langdir):

206 def get_lang_struct(lang, langcode, langname, langfile_mtime): 207 if lang == 'default': 208 real_lang = langcode.lower() 209 else: 210 real_lang = lang 211 (prules_langcode, 212 nplurals, 213 get_plural_id, 214 construct_plural_form 215 ) = PLURAL_RULES.get(real_lang[:2], ('default', 216 DEFAULT_NPLURALS, 217 DEFAULT_GET_PLURAL_ID, 218 DEFAULT_CONSTRUCT_PLURAL_FORM)) 219 if prules_langcode != 'default': 220 (pluraldict_fname, 221 pluraldict_mtime) = plurals.get(real_lang, 222 plurals.get(real_lang[:2], 223 ('plural-%s.py' % real_lang, 0))) 224 else: 225 pluraldict_fname = None 226 pluraldict_mtime = 0 227 return (langcode, # language code from !langcode! 228 langname, 229 # language name in national spelling from !langname! 230 langfile_mtime, # m_time of language file 231 pluraldict_fname, # name of plural dictionary file or None (when default.py is not exist) 232 pluraldict_mtime, # m_time of plural dictionary file or 0 if file is not exist 233 prules_langcode, # code of plural rules language or 'default' 234 nplurals, # nplurals for current language 235 get_plural_id, # get_plural_id() for current language 236 construct_plural_form) # construct_plural_form() for current language

237 238 plurals = {} 239 flist = oslistdir(langdir) if isdir(langdir) else [] 240 241 # scan languages directory for plural dict files: 242 for pname in flist: 243 if regex_plural_file.match(pname): 244 plurals[pname[7:-3]] = (pname, 245 ostat(pjoin(langdir, pname)).st_mtime) 246 langs = {} 247 # scan languages directory for langfiles: 248 for fname in flist: 249 if regex_langfile.match(fname) or fname == 'default.py': 250 fname_with_path = pjoin(langdir, fname) 251 d = read_dict(fname_with_path) 252 lang = fname[:-3] 253 langcode = d.get('!langcode!', lang if lang != 'default' 254 else DEFAULT_LANGUAGE) 255 langname = d.get('!langname!', langcode) 256 langfile_mtime = ostat(fname_with_path).st_mtime 257 langs[lang] = get_lang_struct(lang, langcode, 258 langname, langfile_mtime) 259 if 'default' not in langs: 260 # if default.py is not found, 261 # add DEFAULT_LANGUAGE as default language: 262 langs['default'] = get_lang_struct('default', DEFAULT_LANGUAGE, 263 DEFAULT_LANGUAGE_NAME, 0) 264 deflang = langs['default'] 265 deflangcode = deflang[0] 266 if deflangcode not in langs: 267 # create language from default.py: 268 langs[deflangcode] = deflang[:2] + (0,) + deflang[3:] 269 270 return langs 271 272

273 -def read_possible_languages(langpath):

274 return getcfs('langs:' + langpath, langpath, 275 lambda: read_possible_languages_aux(langpath))

276 277

278 -def read_plural_dict_aux(filename):

279 lang_text = read_locked(filename).replace('\r\n', '\n') 280 try: 281 return eval(lang_text) or {} 282 except Exception: 283 e = sys.exc_info()[1] 284 status = 'Syntax error in %s (%s)' % (filename, e) 285 logging.error(status) 286 return {'__corrupted__': status}

287 288

289 -def read_plural_dict(filename):

290 return getcfs('plurals:' + filename, filename, 291 lambda: read_plural_dict_aux(filename))

292 293

294 -def write_plural_dict(filename, contents):

295 if '__corrupted__' in contents: 296 return 297 try: 298 fp = LockedFile(filename, 'w') 299 fp.write('#!/usr/bin/env python\n{\n# "singular form (0)": ["first plural form (1)", "second plural form (2)", ...],\n') 300 # coding: utf8\n{\n') 301 for key in sorted(contents, lambda x, y: cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())): 302 forms = '[' + ','.join([repr(Utf8(form)) 303 for form in contents[key]]) + ']' 304 fp.write('%s: %s,\n' % (repr(Utf8(key)), forms)) 305 fp.write('}\n') 306 except (IOError, OSError): 307 if not is_gae: 308 logging.warning('Unable to write to file %s' % filename) 309 return 310 finally: 311 fp.close()

312 313

314 -def write_dict(filename, contents):

315 if '__corrupted__' in contents: 316 return 317 try: 318 fp = LockedFile(filename, 'w') 319 except (IOError, OSError): 320 if not settings.global_settings.web2py_runtime_gae: 321 logging.warning('Unable to write to file %s' % filename) 322 return 323 fp.write('# coding: utf8\n{\n') 324 for key in sorted(contents, lambda x, y: cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())): 325 fp.write('%s: %s,\n' % (repr(Utf8(key)), repr(Utf8(contents[key])))) 326 fp.write('}\n') 327 fp.close()

328 329

330 -class lazyT(object):

331 """ 332 never to be called explicitly, returned by 333 translator.__call__() or translator.M() 334 """ 335 m = s = T = f = t = None 336 M = is_copy = False 337

338 - def __init__( 339 self, 340 message, 341 symbols={}, 342 T=None, 343 filter=None, 344 ftag=None, 345 M=False 346 ):

347 if isinstance(message, lazyT): 348 self.m = message.m 349 self.s = message.s 350 self.T = message.T 351 self.f = message.f 352 self.t = message.t 353 self.M = message.M 354 self.is_copy = True 355 else: 356 self.m = message 357 self.s = symbols 358 self.T = T 359 self.f = filter 360 self.t = ftag 361 self.M = M 362 self.is_copy = False

363

364 - def __repr__(self):

365 return "<lazyT %s>" % (repr(Utf8(self.m)), )

366

367 - def __str__(self):

368 return str(self.T.apply_filter(self.m, self.s, self.f, self.t) if self.M else 369 self.T.translate(self.m, self.s))

370

371 - def __eq__(self, other):

372 return str(self) == str(other)

373

374 - def __ne__(self, other):

375 return str(self) != str(other)

376

377 - def __add__(self, other):

378 return '%s%s' % (self, other)

379

380 - def __radd__(self, other):

381 return '%s%s' % (other, self)

382

383 - def __mul__(self, other):

384 return str(self) * other

385

386 - def __cmp__(self, other):

387 return cmp(str(self), str(other))

388

389 - def __hash__(self):

390 return hash(str(self))

391

392 - def __getattr__(self, name):

393 return getattr(str(self), name)

394

395 - def __getitem__(self, i):

396 return str(self)[i]

397

398 - def __getslice__(self, i, j):

399 return str(self)[i:j]

400

401 - def __iter__(self):

402 for c in str(self): 403 yield c

404

405 - def __len__(self):

406 return len(str(self))

407

408 - def xml(self):

409 return str(self) if self.M else escape(str(self))

410

411 - def encode(self, *a, **b):

412 return str(self).encode(*a, **b)

413

414 - def decode(self, *a, **b):

415 return str(self).decode(*a, **b)

416

417 - def read(self):

418 return str(self)

419

420 - def __mod__(self, symbols):

421 if self.is_copy: 422 return lazyT(self) 423 return lazyT(self.m, symbols, self.T, self.f, self.t, self.M)

424

425 -def pickle_lazyT(c):

426 return str, (c.xml(),)

427 428 copy_reg.pickle(lazyT, pickle_lazyT) 429

430 -class translator(object):

431 """ 432 this class is instantiated by gluon.compileapp.build_environment 433 as the T object 434 :: 435 T.force(None) # turns off translation 436 T.force('fr, it') # forces web2py to translate using fr.py or it.py 437 438 T(\"Hello World\") # translates \"Hello World\" using the selected file 439 440 notice 1: there is no need to force since, by default, T uses 441 http_accept_language to determine a translation file. 442 notice 2: 443 en and en-en are considered different languages! 444 notice 3: 445 if language xx-yy is not found force() probes other similar 446 languages using such algorithm: 447 xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py 448 """ 449

450 - def __init__(self, langpath, http_accept_language):

451 self.langpath = langpath 452 self.http_accept_language = http_accept_language 453 self.is_writable = not is_gae 454 # filled in self.force(): 455 #------------------------ 456 # self.cache 457 # self.accepted_language 458 # self.language_file 459 # self.plural_language 460 # self.nplurals 461 # self.get_plural_id 462 # self.construct_plural_form 463 # self.plural_file 464 # self.plural_dict 465 # self.requested_languages 466 #---------------------------------------- 467 # filled in self.set_current_languages(): 468 #---------------------------------------- 469 # self.default_language_file 470 # self.default_t 471 # self.current_languages 472 self.set_current_languages() 473 self.lazy = True 474 self.otherTs = {} 475 self.filter = markmin 476 self.ftag = 'markmin' 477 478 self.ns = None

479

480 - def get_possible_languages_info(self, lang=None):

481 """ 482 return info for selected language or dictionary with all 483 possible languages info from APP/languages/*.py 484 args: 485 *lang* (str): language 486 returns: 487 if *lang* is defined: 488 return tuple(langcode, langname, langfile_mtime, 489 pluraldict_fname, pluraldict_mtime, 490 prules_langcode, nplurals, 491 get_plural_id, construct_plural_form) 492 or None 493 494 if *lang* is NOT defined: 495 returns dictionary with all possible languages: 496 { langcode(from filename): 497 ( langcode, # language code from !langcode! 498 langname, 499 # language name in national spelling from !langname! 500 langfile_mtime, # m_time of language file 501 pluraldict_fname,# name of plural dictionary file or None (when default.py is not exist) 502 pluraldict_mtime,# m_time of plural dictionary file or 0 if file is not exist 503 prules_langcode, # code of plural rules language or 'default' 504 nplurals, # nplurals for current language 505 get_plural_id, # get_plural_id() for current language 506 construct_plural_form) # construct_plural_form() for current language 507 } 508 """ 509 info = read_possible_languages(self.langpath) 510 if lang: 511 info = info.get(lang) 512 return info

513

514 - def get_possible_languages(self):

515 """ get list of all possible languages for current applications """ 516 return list(set(self.current_languages + 517 [lang for lang in read_possible_languages(self.langpath).iterkeys() 518 if lang != 'default']))

519

520 - def set_current_languages(self, *languages):

521 """ 522 set current AKA "default" languages 523 setting one of this languages makes force() function 524 turn translation off to use default language 525 """ 526 if len(languages) == 1 and isinstance( 527 languages[0], (tuple, list)): 528 languages = languages[0] 529 if not languages or languages[0] is None: 530 # set default language from default.py/DEFAULT_LANGUAGE 531 pl_info = self.get_possible_languages_info('default') 532 if pl_info[2] == 0: # langfile_mtime 533 # if languages/default.py is not found 534 self.default_language_file = self.langpath 535 self.default_t = {} 536 self.current_languages = [DEFAULT_LANGUAGE] 537 else: 538 self.default_language_file = pjoin(self.langpath, 539 'default.py') 540 self.default_t = read_dict(self.default_language_file) 541 self.current_languages = [pl_info[0]] # !langcode! 542 else: 543 self.current_languages = list(languages) 544 self.force(self.http_accept_language)

545

546 - def plural(self, word, n):

547 """ get plural form of word for number *n* 548 NOTE: *word" MUST be defined in current language 549 (T.accepted_language) 550 551 invoked from T()/T.M() in %%{} tag 552 args: 553 word (str): word in singular 554 n (numeric): number plural form created for 555 556 returns: 557 (str): word in appropriate singular/plural form 558 """ 559 if int(n) == 1: 560 return word 561 elif word: 562 id = self.get_plural_id(abs(int(n))) 563 # id = 0 singular form 564 # id = 1 first plural form 565 # id = 2 second plural form 566 # etc. 567 if id != 0: 568 forms = self.plural_dict.get(word, []) 569 if len(forms) >= id: 570 # have this plural form: 571 return forms[id - 1] 572 else: 573 # guessing this plural form 574 forms += [''] * (self.nplurals - len(forms) - 1) 575 form = self.construct_plural_form(word, id) 576 forms[id - 1] = form 577 self.plural_dict[word] = forms 578 if self.is_writable and self.plural_file: 579 write_plural_dict(self.plural_file, 580 self.plural_dict) 581 return form 582 return word

583

584 - def force(self, *languages):

585 """ 586 587 select language(s) for translation 588 589 if a list of languages is passed as a parameter, 590 first language from this list that matches the ones 591 from the possible_languages dictionary will be 592 selected 593 594 default language will be selected if none 595 of them matches possible_languages. 596 """ 597 pl_info = read_possible_languages(self.langpath) 598 599 def set_plural(language): 600 """ 601 initialize plural forms subsystem 602 """ 603 lang_info = pl_info.get(language) 604 if lang_info: 605 (pname, 606 pmtime, 607 self.plural_language, 608 self.nplurals, 609 self.get_plural_id, 610 self.construct_plural_form 611 ) = lang_info[3:] 612 pdict = {} 613 if pname: 614 pname = pjoin(self.langpath, pname) 615 if pmtime != 0: 616 pdict = read_plural_dict(pname) 617 self.plural_file = pname 618 self.plural_dict = pdict 619 else: 620 self.plural_language = 'default' 621 self.nplurals = DEFAULT_NPLURALS 622 self.get_plural_id = DEFAULT_GET_PLURAL_ID 623 self.construct_plural_form = DEFAULT_CONSTRUCT_PLURAL_FORM 624 self.plural_file = None 625 self.plural_dict = {}

626 language = '' 627 if len(languages) == 1 and isinstance(languages[0], str): 628 languages = regex_language.findall(languages[0].lower()) 629 elif not languages or languages[0] is None: 630 languages = [] 631 self.requested_languages = languages = tuple(languages) 632 if languages: 633 all_languages = set(lang for lang in pl_info.iterkeys() 634 if lang != 'default') \ 635 | set(self.current_languages) 636 for lang in languages: 637 # compare "aa-bb" | "aa" from *language* parameter 638 # with strings from langlist using such alghorythm: 639 # xx-yy.py -> xx.py -> xx*.py 640 lang5 = lang[:5] 641 if lang5 in all_languages: 642 language = lang5 643 else: 644 lang2 = lang[:2] 645 if len(lang5) > 2 and lang2 in all_languages: 646 language = lang2 647 else: 648 for l in all_languages: 649 if l[:2] == lang2: 650 language = l 651 if language: 652 if language in self.current_languages: 653 break 654 self.language_file = pjoin(self.langpath, language + '.py') 655 self.t = read_dict(self.language_file) 656 self.cache = global_language_cache.setdefault( 657 self.language_file, 658 ({}, RLock())) 659 set_plural(language) 660 self.accepted_language = language 661 return languages 662 self.accepted_language = language or self.current_languages[0] 663 self.language_file = self.default_language_file 664 self.cache = global_language_cache.setdefault(self.language_file, 665 ({}, RLock())) 666 self.t = self.default_t 667 set_plural(self.accepted_language) 668 return languages

669

670 - def __call__(self, message, symbols={}, language=None, lazy=None, ns=None):

671 """ 672 get cached translated plain text message with inserted parameters(symbols) 673 if lazy==True lazyT object is returned 674 """ 675 if lazy is None: 676 lazy = self.lazy 677 if not language and not ns: 678 if lazy: 679 return lazyT(message, symbols, self) 680 else: 681 return self.translate(message, symbols) 682 else: 683 if ns: 684 if ns != self.ns: 685 self.langpath = os.path.join(self.langpath, ns) 686 if self.ns is None: 687 self.ns = ns 688 otherT = self.__get_otherT__(language, ns) 689 return otherT(message, symbols, lazy=lazy)

690

691 - def __get_otherT__(self, language=None, namespace=None):

692 if not language and not namespace: 693 raise Exception('Incorrect parameters') 694 695 if namespace: 696 if language: 697 index = '%s/%s' % (namespace, language) 698 else: 699 index = namespace 700 else: 701 index = language 702 try: 703 otherT = self.otherTs[index] 704 except KeyError: 705 otherT = self.otherTs[index] = translator(self.langpath, \ 706 self.http_accept_language) 707 if language: 708 otherT.force(language) 709 return otherT

710

711 - def apply_filter(self, message, symbols={}, filter=None, ftag=None):

712 def get_tr(message, prefix, filter): 713 s = self.get_t(message, prefix) 714 return filter(s) if filter else self.filter(s)

715 if filter: 716 prefix = '@' + (ftag or 'userdef') + '\x01' 717 else: 718 prefix = '@' + self.ftag + '\x01' 719 message = get_from_cache( 720 self.cache, prefix + message, 721 lambda: get_tr(message, prefix, filter)) 722 if symbols or symbols == 0 or symbols == "": 723 if isinstance(symbols, dict): 724 symbols.update( 725 (key, xmlescape(value).translate(ttab_in)) 726 for key, value in symbols.iteritems() 727 if not isinstance(value, NUMBERS)) 728 else: 729 if not isinstance(symbols, tuple): 730 symbols = (symbols,) 731 symbols = tuple( 732 value if isinstance(value, NUMBERS) 733 else xmlescape(value).translate(ttab_in) 734 for value in symbols) 735 message = self.params_substitution(message, symbols) 736 return XML(message.translate(ttab_out)) 737

738 - def M(self, message, symbols={}, language=None, 739 lazy=None, filter=None, ftag=None, ns=None):

740 """ 741 get cached translated markmin-message with inserted parametes 742 if lazy==True lazyT object is returned 743 """ 744 if lazy is None: 745 lazy = self.lazy 746 if not language and not ns: 747 if lazy: 748 return lazyT(message, symbols, self, filter, ftag, True) 749 else: 750 return self.apply_filter(message, symbols, filter, ftag) 751 else: 752 if ns: 753 self.langpath = os.path.join(self.langpath, ns) 754 otherT = self.__get_otherT__(language, ns) 755 return otherT.M(message, symbols, lazy=lazy)

756

757 - def get_t(self, message, prefix=''):

758 """ 759 user ## to add a comment into a translation string 760 the comment can be useful do discriminate different possible 761 translations for the same string (for example different locations) 762 763 T(' hello world ') -> ' hello world ' 764 T(' hello world ## token') -> ' hello world ' 765 T('hello ## world## token') -> 'hello ## world' 766 767 the ## notation is ignored in multiline strings and strings that 768 start with ##. this is to allow markmin syntax to be translated 769 """ 770 if isinstance(message, unicode): 771 message = message.encode('utf8') 772 if isinstance(prefix, unicode): 773 prefix = prefix.encode('utf8') 774 key = prefix + message 775 mt = self.t.get(key, None) 776 if mt is not None: 777 return mt 778 # we did not find a translation 779 if message.find('##') > 0 and not '\n' in message: 780 # remove comments 781 message = message.rsplit('##', 1)[0] 782 # guess translation same as original 783 self.t[key] = mt = self.default_t.get(key, message) 784 # update language file for latter translation 785 if self.is_writable and self.language_file != self.default_language_file: 786 write_dict(self.language_file, self.t) 787 return regex_backslash.sub( 788 lambda m: m.group(1).translate(ttab_in), mt)

789

790 - def params_substitution(self, message, symbols):

791 """ 792 substitute parameters from symbols into message using %. 793 also parse %%{} placeholders for plural-forms processing. 794 returns: string with parameters 795 NOTE: *symbols* MUST BE OR tuple OR dict of parameters! 796 """ 797 def sub_plural(m): 798 """string in %{} is transformed by this rules: 799 If string starts with \\, ! or ? such transformations 800 take place: 801 802 "!string of words" -> "String of word" (Capitalize) 803 "!!string of words" -> "String Of Word" (Title) 804 "!!!string of words" -> "STRING OF WORD" (Upper) 805 "\\!string of words" -> "!string of word" 806 (remove \\ and disable transformations) 807 "?word?number" -> "word" (return word, if number == 1) 808 "?number" or "??number" -> "" (remove number, 809 if number == 1) 810 "?word?number" -> "number" (if number != 1) 811 """ 812 def sub_tuple(m): 813 """ word[number], !word[number], !!word[number], !!!word[number] 814 word, !word, !!word, !!!word, ?word?number, ??number, ?number 815 ?word?word[number], ?word?[number], ??word[number] 816 """ 817 w, i = m.group('w', 'i') 818 c = w[0] 819 if c not in '!?': 820 return self.plural(w, symbols[int(i or 0)]) 821 elif c == '?': 822 (p1, sep, p2) = w[1:].partition("?") 823 part1 = p1 if sep else "" 824 (part2, sep, part3) = (p2 if sep else p1).partition("?") 825 if not sep: 826 part3 = part2 827 if i is None: 828 # ?[word]?number[?number] or ?number 829 if not part2: 830 return m.group(0) 831 num = int(part2) 832 else: 833 # ?[word]?word2[?word3][number] 834 num = int(symbols[int(i or 0)]) 835 return part1 if num == 1 else part3 if num == 0 else part2 836 elif w.startswith('!!!'): 837 word = w[3:] 838 fun = upper_fun 839 elif w.startswith('!!'): 840 word = w[2:] 841 fun = title_fun 842 else: 843 word = w[1:] 844 fun = cap_fun 845 if i is not None: 846 return fun(self.plural(word, symbols[int(i)])) 847 return fun(word)

848 849 def sub_dict(m): 850 """ word(var), !word(var), !!word(var), !!!word(var) 851 word(num), !word(num), !!word(num), !!!word(num) 852 ?word2(var), ?word1?word2(var), ?word1?word2?word0(var) 853 ?word2(num), ?word1?word2(num), ?word1?word2?word0(num) 854 """ 855 w, n = m.group('w', 'n') 856 c = w[0] 857 n = int(n) if n.isdigit() else symbols[n] 858 if c not in '!?': 859 return self.plural(w, n) 860 elif c == '?': 861 # ?[word1]?word2[?word0](var or num), ?[word1]?word2(var or num) or ?word2(var or num) 862 (p1, sep, p2) = w[1:].partition("?") 863 part1 = p1 if sep else "" 864 (part2, sep, part3) = (p2 if sep else p1).partition("?") 865 if not sep: 866 part3 = part2 867 num = int(n) 868 return part1 if num == 1 else part3 if num == 0 else part2 869 elif w.startswith('!!!'): 870 word = w[3:] 871 fun = upper_fun 872 elif w.startswith('!!'): 873 word = w[2:] 874 fun = title_fun 875 else: 876 word = w[1:] 877 fun = cap_fun 878 return fun(self.plural(word, n)) 879 880 s = m.group(1) 881 part = regex_plural_tuple.sub(sub_tuple, s) 882 if part == s: 883 part = regex_plural_dict.sub(sub_dict, s) 884 if part == s: 885 return m.group(0) 886 return part 887 message = message % symbols 888 message = regex_plural.sub(sub_plural, message) 889 return message 890

891 - def translate(self, message, symbols):

892 """ 893 get cached translated message with inserted parameters(symbols) 894 """ 895 message = get_from_cache(self.cache, message, 896 lambda: self.get_t(message)) 897 if symbols or symbols == 0 or symbols == "": 898 if isinstance(symbols, dict): 899 symbols.update( 900 (key, str(value).translate(ttab_in)) 901 for key, value in symbols.iteritems() 902 if not isinstance(value, NUMBERS)) 903 else: 904 if not isinstance(symbols, tuple): 905 symbols = (symbols,) 906 symbols = tuple( 907 value if isinstance(value, NUMBERS) 908 else str(value).translate(ttab_in) 909 for value in symbols) 910 message = self.params_substitution(message, symbols) 911 return message.translate(ttab_out)

912 913

914 -def findT(path, language=DEFAULT_LANGUAGE):

915 """ 916 must be run by the admin app 917 """ 918 lang_file = pjoin(path, 'languages', language + '.py') 919 sentences = read_dict(lang_file) 920 mp = pjoin(path, 'models') 921 cp = pjoin(path, 'controllers') 922 vp = pjoin(path, 'views') 923 mop = pjoin(path, 'modules') 924 for filename in \ 925 listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\ 926 + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0): 927 data = read_locked(filename) 928 items = regex_translate.findall(data) 929 for item in items: 930 try: 931 message = safe_eval(item) 932 except: 933 continue # silently ignore inproperly formatted strings 934 if not message.startswith('#') and not '\n' in message: 935 tokens = message.rsplit('##', 1) 936 else: 937 # this allows markmin syntax in translations 938 tokens = [message] 939 if len(tokens) == 2: 940 message = tokens[0].strip() + '##' + tokens[1].strip() 941 if message and not message in sentences: 942 sentences[message] = message 943 if not '!langcode!' in sentences: 944 sentences['!langcode!'] = ( 945 DEFAULT_LANGUAGE if language in ('default', DEFAULT_LANGUAGE) else language) 946 if not '!langname!' in sentences: 947 sentences['!langname!'] = ( 948 DEFAULT_LANGUAGE_NAME if language in ('default', DEFAULT_LANGUAGE) 949 else sentences['!langcode!']) 950 write_dict(lang_file, sentences)

951

952 -def update_all_languages(application_path):

953 path = pjoin(application_path, 'languages/') 954 for language in oslistdir(path): 955 if regex_langfile.match(language): 956 findT(application_path, language[:-3])

957 958 959 if __name__ == '__main__': 960 import doctest 961 doctest.testmod() 962

Source Code for Module gluon.languages