Bläddra i källkod

fixes to translations, added building of translation assets

version-14
Rushabh Mehta 11 år sedan
förälder
incheckning
f647430ebc
11 ändrade filer med 83 tillägg och 437 borttagningar
  1. +2
    -0
      webnotes/__init__.py
  2. +3
    -2
      webnotes/build.py
  3. +2
    -2
      webnotes/cli.py
  4. +14
    -2
      webnotes/client.py
  5. +8
    -15
      webnotes/public/js/wn/assets.js
  6. +4
    -2
      webnotes/public/js/wn/request.js
  7. +2
    -4
      webnotes/pythonrc.py
  8. +2
    -1
      webnotes/tests/test_translation.py
  9. +43
    -407
      webnotes/translate.py
  10. +1
    -1
      webnotes/widgets/page.py
  11. +2
    -1
      webnotes/widgets/query_report.py

+ 2
- 0
webnotes/__init__.py Visa fil

@@ -257,7 +257,9 @@ def clear_cache(user=None, doctype=None):
elif user:
clear_cache(user)
else: # everything
import translate
clear_cache()
translate.clear_cache()
reset_metadata_version()
def get_roles(username=None):


+ 3
- 2
webnotes/build.py Visa fil

@@ -10,12 +10,14 @@ Build the `public` folders and setup languages

import os, sys, webnotes, json
from cssmin import cssmin
import webnotes.translate

def bundle(no_compress):
"""concat / minify js files"""
# build js files
make_site_public_dirs()
build(no_compress)
webnotes.translate.clear_cache()
def watch(no_compress):
"""watch and rebuild if necessary"""
@@ -53,8 +55,7 @@ def make_site_public_dirs():
site_public_assets = os.path.join(site_public_path, "assets")
if not os.path.exists(site_public_assets):
os.symlink(os.path.abspath(assets_path), site_public_assets)

def clear_pyc_files():
from webnotes.utils import get_base_path
for path, folders, files in os.walk(get_base_path()):


+ 2
- 2
webnotes/cli.py Visa fil

@@ -563,13 +563,13 @@ def mysql():
webnotes.destroy()

@cmd
def python():
def python(site):
import webnotes
import commands, os
python = commands.getoutput('which python')
if site:
os.environ["site"] = site
os.environ["PYTHONSTARTUP"] = os.path.join(os.path.dirname(__file__), "pythonrc.py")
os.environ["PYTHONSTARTUP"] = os.path.join(os.path.dirname(webnotes.__file__), "pythonrc.py")
os.execv(python, [python])
webnotes.destroy()



+ 14
- 2
webnotes/client.py Visa fil

@@ -6,7 +6,7 @@ import webnotes
from webnotes import _
import webnotes.model
import webnotes.utils
import json
import json, os

@webnotes.whitelist()
def get(doctype, name=None, filters=None):
@@ -118,4 +118,16 @@ def bulk_update(docs):
@webnotes.whitelist()
def has_permission(doctype, docname, perm_type="read"):
# perm_type can be one of read, write, create, submit, cancel, report
return {"has_permission": webnotes.has_permission(doctype, perm_type.lower(), docname)}
return {"has_permission": webnotes.has_permission(doctype, perm_type.lower(), docname)}
@webnotes.whitelist()
def get_js(src):
contentpath = os.path.join(webnotes.local.sites_path, src)
with open(contentpath, "r") as srcfile:
code = srcfile.read()
if webnotes.local.lang != "en":
code += "\n\n$.extend(wn._messages, {})".format(json.dumps(\
webnotes.get_lang_dict("jsfile", contentpath)))
return code

+ 8
- 15
webnotes/public/js/wn/assets.js Visa fil

@@ -13,10 +13,7 @@ wn.require = function(items) {

for(var i=0; i< l; i++) {
var src = items[i];
//if(!(src in wn.assets.executed_)) {
// check if available in localstorage
wn.assets.execute(src);
//}
}
};

@@ -83,23 +80,19 @@ wn.assets = {
// load an asset via
load: function(src) {
// this is virtual page load, only get the the source
// *without* the template
var t = src;
// *without* the template
wn.set_loading();

$.ajax({
url: t,
data: {
q: Math.floor(Math.random()*1000)
wn.call({
method:"webnotes.client.get_js",
args: {
"src": src
},
dataType: 'text',
success: function(txt) {
// add it to localstorage
wn.assets.add(src, txt);
callback: function(r) {
wn.assets.add(src, r.message);
},
async: false
});
})
wn.done_loading();
},


+ 4
- 2
webnotes/public/js/wn/request.js Visa fil

@@ -40,7 +40,8 @@ wn.call = function(opts) {
btn: opts.btn,
freeze: opts.freeze,
show_spinner: !opts.no_spinner,
progress_bar: opts.progress_bar
progress_bar: opts.progress_bar,
async: opts.async
});
}

@@ -64,7 +65,8 @@ wn.request.call = function(opts) {
wn.request.cleanup(opts, {});
show_alert(wn._("Unable to complete request: ") + textStatus)
opts.error && opts.error(xhr)
}
},
async: opts.async
};
wn.last_request = ajax_args.data;


+ 2
- 4
webnotes/pythonrc.py Visa fil

@@ -5,8 +5,6 @@

from __future__ import unicode_literals

import os, sys
sys.path = [".", "lib", "app"] + sys.path

import os
import webnotes
webnotes.connect(site=os.environ.get("site"))
webnotes.connect(site=os.environ.get("site"))

+ 2
- 1
webnotes/tests/test_translation.py Visa fil

@@ -48,7 +48,8 @@ class TestTranslations(unittest.TestCase):
def test_write_csv(self):
tpath = webnotes.get_pymodule_path("webnotes", "translations", "de.csv")
os.remove(tpath)
if os.path.exists(tpath):
os.remove(tpath)
webnotes.translate.write_translations_file("webnotes", "de")
self.assertTrue(os.path.exists(tpath))
self.assertEquals(dict(webnotes.translate.read_csv_file(tpath)).get("Row"), "Reihe")


+ 43
- 407
webnotes/translate.py Visa fil

@@ -16,7 +16,7 @@ Contributing:
# wn.require
# webnotes._

import webnotes, os, re, codecs
import webnotes, os, re, codecs, json

def get_all_languages():
return [a.split()[0] for a in get_lang_info()]
@@ -41,17 +41,37 @@ def write_translations_file(app, lang, full_dict=None):

def get_dict(fortype, name=None):
fortype = fortype.lower()
if fortype=="doctype":
messages = get_messages_from_doctype(name)
elif fortype=="page":
messages = get_messages_from_page(name)
elif fortype=="report":
messages = get_messages_from_report(name)
elif fortype=="include":
messages = get_messages_from_include_files()
cache = webnotes.cache()
cache_key = "translation_assets:" + webnotes.local.lang
asset_key = fortype + ":" + name
translation_assets = cache.get_value(cache_key) or {}
if not asset_key in translation_assets:
if fortype=="doctype":
messages = get_messages_from_doctype(name)
elif fortype=="page":
messages = get_messages_from_page(name)
elif fortype=="report":
messages = get_messages_from_report(name)
elif fortype=="include":
messages = get_messages_from_include_files()
elif fortype=="jsfile":
messages = get_messages_from_file(name)
translation_assets[asset_key] = make_dict_from_messages(messages)
cache.set_value(cache_key, translation_assets)
return translation_assets[asset_key]
def add_lang_dict(code):
messages = extract_messages_from_code(code)
code += "\n\n$.extend(wn._messages, %s)" % json.dumps(make_dict_from_messages(messages))
return code

def make_dict_from_messages(messages, full_dict=None):
out = {}
full_dict = get_full_dict(webnotes.local.lang)
if full_dict==None:
full_dict = get_full_dict(webnotes.local.lang)
for m in messages:
if m in full_dict:
@@ -60,7 +80,7 @@ def get_dict(fortype, name=None):
return out

def get_lang_js(fortype, name):
return "\n\n$.extend(wn._messages, %s)" % get_dict(fortype, name)
return "\n\n$.extend(wn._messages, %s)" % json.dumps(get_dict(fortype, name))

def get_full_dict(lang):
if lang == "en": return {}
@@ -80,6 +100,7 @@ def clear_cache():
cache.delete_value("langinfo")
for lang in get_all_languages():
cache.delete_value("lang:" + lang)
cache.delete_value("translation_assets:" + lang)
def get_messages_for_app(app):
messages = []
@@ -174,14 +195,18 @@ def get_messages_from_include_files(app_name=None):
def get_messages_from_file(path):
"""get list of messages from a code file"""
messages = []
if os.path.exists(path):
with open(path, 'r') as sourcefile:
txt = sourcefile.read()
messages += re.findall('_\("([^"]*)".*\)', txt)
messages += re.findall("_\('([^']*)'.*\)", txt)
messages += re.findall('_\("{3}([^"]*)"{3}.*\)', txt, re.S)
return extract_messages_from_code(sourcefile.read(), path.endswith(".py"))
else:
return []

def extract_messages_from_code(code, is_py=False):
messages = []
messages += re.findall('_\("([^"]*)"\)', code)
messages += re.findall("_\('([^']*)'\)", code)
if is_py:
messages += re.findall('_\("{3}([^"]*)"{3}.*\)', code, re.S)
return messages
def clean(messages):
@@ -257,393 +282,4 @@ def google_translate(lang, untranslated):
return dict(zip(untranslated, translated))
else:
print "unable to translate"
return {}

# import webnotes
# import os
# import codecs
# import json
# import re
# from csv import reader
# from webnotes.modules import get_doc_path,get_doctype_module
# from webnotes.utils import cstr
#
# def translate(lang=None):
# languages = [lang]
# if lang=="all" or lang==None:
# languages = get_all_languages()
#
# print "Extracting / updating translatable strings..."
# build_message_files()
#
# print "Compiling messages in one file..."
# export_messages(lang, '_lang_tmp.csv')
#
# for lang in languages:
# if lang != "en":
# filename = 'app/translations/'+lang+'.csv'
# print "For " + lang + ":"
# print "Translating via Google Translate..."
# google_translate(lang, '_lang_tmp.csv', filename)
# print "Updating language files..."
# import_messages(lang, filename)
#
# print "Deleting temp file..."
# os.remove('_lang_tmp.csv')
#
# def get_all_languages():
# languages = [a.split()[0] for a in \
# webnotes.get_file_items(os.path.join(webnotes.local.sites_path, "languages.txt"))]
#
# def get_lang_dict():
# languages_path = os.path.join(os.path.dirname(webnotes.__file__), "data", "languages.json")
# if os.path.exists(languages_path):
# with open(languages_path, "r") as langfile:
# return json.loads(langfile.read())
# else: return {}
#
# def update_translations():
# """
# compare language file timestamps with last updated timestamps in `.wnf-lang-status`
# if timestamps are missing / changed, build new `.json` files in the `lang folders`
# """
# langstatus = {}
# languages = get_all_languages()
# message_updated = False
# status_file_path = "app/.wnf-lang-status"
#
# if not os.path.exists(os.path.join('app', 'translations')):
# return
# if os.path.exists(status_file_path):
# with open(status_file_path, "r") as langstatusfile:
# langstatus = eval(langstatusfile.read())
#
# for lang in languages:
# filename = os.path.join('app', 'translations', lang + '.csv')
# if langstatus.get(lang, None)!=os.path.getmtime(filename):
# print "Setting up lang files for " + lang + "..."
# if not message_updated:
# print "Extracting / updating translatable strings..."
# build_message_files()
# message_updated = True
# print "Writing translations..."
# import_messages(lang, filename)
# langstatus[lang] = os.path.getmtime(filename)
#
# with open(status_file_path, "w") as langstatusfile:
# langstatus = langstatusfile.write(str(langstatus))
#
# def build_message_files():
# """build from doctypes, pages, database and framework"""
# if not webnotes.conn:
# webnotes.connect()
#
# for app in webnotes.get_all_apps(True):
# build_for_pages(webnotes.get_pymodule_path(app))
# build_from_doctype_code(webnotes.get_pymodule_path(app))
#
# #reports
# build_from_query_report()
#
# # doctype
# build_from_database()
#
# build_for_framework('lib/webnotes', 'py', with_doctype_names=True)
# build_for_framework('lib/public/js/wn', 'js')
# build_for_framework('app/public/js', 'js', with_doctype_names=True)
#
# def build_for_pages(path):
# """make locale files for framework py and js (all)"""
# messages = []
# for (basepath, folders, files) in os.walk(path):
# if 'locale' in folders: folders.remove('locale')
# if os.path.basename(os.path.dirname(basepath))=="page":
# messages_js, messages_py = [], []
# for fname in files:
# fname = cstr(fname)
# if fname.endswith('.js'):
# messages_js += get_message_list(os.path.join(basepath, fname))
# if fname.endswith('.py'):
# messages_py += get_message_list(os.path.join(basepath, fname))
# if messages_js:
# write_messages_file(basepath, messages_js, "js")
# if messages_py:
# write_messages_file(basepath, messages_py, "py")
#
# def build_from_query_report():
# """make locale for the query reports from database and the framework js and py files"""
# import re
# for item in webnotes.conn.sql("""select name, report_name,ref_doctype, query
# from `tabReport`""", as_dict=1):
# messages_js, messages_py = [], []
#
# if item:
# messages_js.append(item.report_name)
# messages_py.append(item.report_name)
# # get the messages from the query using the regex :
# # if we have the string "Production Date:Date:180" in the query then the regex will search for string between " and : .
# # the regex will take "Production Date" and store them into messages
# if item.query :
# messages_query = re.findall('"([^:,^"]*):', item.query)
# messages_js += messages_query
# messages_py += messages_query
#
# module = get_doctype_module(item.ref_doctype)
# if module :
# doctype_path = get_doc_path(module, "Report", item.name)
# if os.path.exists(doctype_path):
# for (basepath, folders, files) in os.walk(doctype_path):
# if 'locale' in folders: folders.remove('locale')
# for fname in files:
# if fname.endswith('.js'):
# messages_js += get_message_list(os.path.join(basepath, fname))
# if fname.endswith('.py'):
# messages_py += get_message_list(os.path.join(basepath, fname))
# break
# write_messages_file(doctype_path, messages_js, 'js')
# write_messages_file(doctype_path, messages_py, 'py')
#
#
# def build_from_database():
# """make doctype labels, names, options, descriptions"""
# def get_select_options(doc):
# if doc.doctype=="DocField" and doc.fieldtype=='Select' and doc.options \
# and not doc.options.startswith("link:") \
# and not doc.options.startswith("attach_files:"):
# return doc.options.split('\n')
# else:
# return []
#
# build_for_doc_from_database(webnotes._dict({
# "doctype": "DocType",
# "module_field": "module",
# "DocType": ["name", "description", "module"],
# "DocField": ["label", "description"],
# "custom": get_select_options
# }))
#
# def build_for_doc_from_database(fields):
# for item in webnotes.conn.sql("""select name from `tab%s`""" % fields.doctype, as_dict=1):
# messages = []
# doclist = webnotes.bean(fields.doctype, item.name).doclist
#
# for doc in doclist:
# if doc.doctype in fields:
# messages += map(lambda x: x in fields[doc.doctype] and doc.fields.get(x) or None,
# doc.fields.keys())
#
# if fields.custom:
# messages += fields.custom(doc)
#
# doc = doclist[0]
# if doc.fields.get(fields.module_field):
# doctype_path = get_doc_path(doc.fields[fields.module_field],
# doc.doctype, doc.name)
# write_messages_file(doctype_path, messages, 'doc')
#
# def build_for_framework(path, mtype, with_doctype_names = False):
# """make locale files for framework py and js (all)"""
# messages = []
# for (basepath, folders, files) in os.walk(path):
# if 'locale' in folders: folders.remove('locale')
# for fname in files:
# fname = cstr(fname)
# if fname.endswith('.' + mtype):
# messages += get_message_list(os.path.join(basepath, fname))
#
#
# # append module & doctype names
# if with_doctype_names:
# for m in webnotes.conn.sql("""select name, module from `tabDocType`"""):
# messages.append(m[0])
# messages.append(m[1])
#
# # append labels from config.json
# config = webnotes.get_config()
# for moduleinfo in config["modules"].values():
# if moduleinfo.get("label"):
# messages.append(moduleinfo["label"])
#
# if messages:
# write_messages_file(path, messages, mtype)
#
# def build_from_doctype_code(path):
# """walk and make locale files in all folders"""
# for (basepath, folders, files) in os.walk(path):
# messagespy = []
# messagesjs = []
# for fname in files:
# fname = cstr(fname)
# if fname.endswith('py'):
# messagespy += get_message_list(os.path.join(basepath, fname))
# if fname.endswith('js'):
# messagesjs += get_message_list(os.path.join(basepath, fname))
#
# if messagespy:
# write_messages_file(basepath, messagespy, 'py')
#
# if messagespy:
# write_messages_file(basepath, messagesjs, 'js')
#
# def get_message_list(path):
# """get list of messages from a code file"""
# import re
# messages = []
# with open(path, 'r') as sourcefile:
# txt = sourcefile.read()
# messages += re.findall('_\("([^"]*)".*\)', txt)
# messages += re.findall("_\('([^']*)'.*\)", txt)
# messages += re.findall('_\("{3}([^"]*)"{3}.*\)', txt, re.S)
#
# return messages
#
# def write_messages_file(path, messages, mtype):
# """write messages to translation file"""
# if not os.path.exists(path):
# return
#
# if not os.path.exists(os.path.join(path, 'locale')):
# os.makedirs(os.path.join(path, 'locale'))
#
# fname = os.path.join(path, 'locale', '_messages_' + mtype + '.json')
# messages = list(set(messages))
# filtered = []
# for m in messages:
# if m and re.search('[a-zA-Z]+', m):
# filtered.append(m)
# with open(fname, 'w') as msgfile:
# msgfile.write(json.dumps(filtered, indent=1))
#
# def export_messages(lang, outfile):
# """get list of all messages"""
# messages = {}
# # extract messages
# for (basepath, folders, files) in os.walk('.'):
# def _get_messages(messages, basepath, mtype):
# mlist = get_messages(basepath, mtype)
# if not mlist:
# return
#
# # update messages with already existing translations
# langdata = get_lang_data(basepath, lang, mtype)
# for m in mlist:
# if not messages.get(m):
# messages[m] = langdata.get(m, "")
#
# if os.path.basename(basepath)=='locale':
# _get_messages(messages, basepath, 'doc')
# _get_messages(messages, basepath, 'py')
# _get_messages(messages, basepath, 'js')
#
# # remove duplicates
# if outfile:
# from csv import writer
# with open(outfile, 'w') as msgfile:
# w = writer(msgfile)
# keys = messages.keys()
# keys.sort()
# for m in keys:
#
# w.writerow([m.encode('utf-8'), messages.get(m, '').encode('utf-8')])
#
# def import_messages(lang, infile):
# """make individual message files for each language"""
# data = dict(get_all_messages_from_file(infile))
#
# for (basepath, folders, files) in os.walk('.'):
# def _update_lang_file(mtype):
# """create a langauge file for the given message type"""
# messages = get_messages(basepath, mtype)
# if not messages: return
#
# # read existing
# langdata = get_lang_data(basepath, lang, mtype)
#
# # update fresh
# for m in messages:
# if data.get(m):
# langdata[m] = data.get(m)
#
# if langdata:
# # write new langfile
# langfilename = os.path.join(basepath, lang + '-' + mtype + '.json')
# with open(langfilename, 'w') as langfile:
# langfile.write(json.dumps(langdata, indent=1, sort_keys=True).encode('utf-8'))
# #print 'wrote ' + langfilename
#
# if os.path.basename(basepath)=='locale':
# # make / update lang files for each type of message file (doc, js, py)
# # example: hi-doc.json, hi-js.json, hi-py.json
# _update_lang_file('doc')
# _update_lang_file('js')
# _update_lang_file('py')
#
# def load_doc_messages(module, doctype, name):
# if webnotes.lang=="en":
# return {}
#
# if not webnotes.local.translated_docs:
# webnotes.local.translated_docs = []
#
# doc_path = get_doc_path(module, doctype, name)
#
# # don't repload the same doc again
# if (webnotes.lang + ":" + doc_path) in webnotes.local.translated_docs:
# return
#
# if not docs_loaded:
# webnotes.local.translate_docs_loaded = []
# webnotes.local.translated_docs.append(webnotes.lang + ":" + doc_path)
#
# webnotes.local.translations.update(get_lang_data(doc_path, None, 'doc'))
#
# def get_lang_data(basepath, lang, mtype):
# """get language dict from langfile"""
#
# # add "locale" folder if reqd
# if os.path.basename(basepath) != 'locale':
# basepath = os.path.join(basepath, 'locale')
#
# if not lang: lang = webnotes.local.lang
#
# path = os.path.join(basepath, lang + '-' + mtype + '.json')
#
# langdata = {}
# if os.path.exists(path):
# with codecs.open(path, 'r', 'utf-8') as langfile:
# langdata = json.loads(langfile.read())
#
# return langdata
#
# def get_messages(basepath, mtype):
# """load list of messages from _message files"""
# # get message list
# path = os.path.join(basepath, '_messages_' + mtype + '.json')
# messages = []
# if os.path.exists(path):
# with open(path, 'r') as msgfile:
# messages = json.loads(msgfile.read())
#
# return messages
#
# def update_lang_js(jscode, path):
# return jscode + "\n\n$.extend(wn._messages, %s)" % \
# json.dumps(get_lang_data(path, webnotes.lang, 'js'))
#
# def get_all_messages_from_file(path):
# with codecs.open(path, 'r', 'utf-8') as msgfile:
# data = msgfile.read()
# data = reader([r.encode('utf-8') for r in data.splitlines()])
# newdata = []
# for row in data:
# newrow = []
# for val in row:
# newrow.append(unicode(val, 'utf-8'))
# newdata.append(newrow)
#
# return newdata
#
return {}

+ 1
- 1
webnotes/widgets/page.py Visa fil

@@ -26,7 +26,7 @@ def getpage():
if has_permission(doclist):
# load translations
if webnotes.lang != "en":
webnotes.response["__messages"] = webnotes.get_lang_dict("page", d.name)
webnotes.response["__messages"] = webnotes.get_lang_dict("page", page)

webnotes.response['docs'] = doclist
else:


+ 2
- 1
webnotes/widgets/query_report.py Visa fil

@@ -63,7 +63,8 @@ def run(report_name, filters=None):
else:
module = webnotes.conn.get_value("DocType", report.ref_doctype, "module")
if report.is_standard=="Yes":
method_name = scrub(module) + ".report." + scrub(report.name) + "." + scrub(report.name) + ".execute"
method_name = webnotes.local.module_app[scrub(module)] + "." + scrub(module) \
+ ".report." + scrub(report.name) + "." + scrub(report.name) + ".execute"
columns, result = webnotes.get_attr(method_name)(filters or {})
result = get_filtered_data(report.ref_doctype, columns, result)


Laddar…
Avbryt
Spara