diff --git a/shared/generate_i18n_gtranslate.py b/shared/generate_i18n_gtranslate.py deleted file mode 100644 index 8f6e6544..00000000 --- a/shared/generate_i18n_gtranslate.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python2.7 - -""" -We want python 2.7 again... -""" - -import io -import re -import json -import sys - -""" -from googletrans import Translator # Use the free webhook -def run_translate(messages, source_language, target_language): - translator = Translator() - _translations = translator.translate(messages, src=source_language, dest=target_language) - result = [] - for translation in _translations: - result.append({ - "source": translation.origin, - "translated": translation.text - }) - return result -""" - - -from google.cloud import translate # Use googles could solution -def run_translate(messages, source_language, target_language): - translate_client = translate.Client() - - # The text to translate - text = u'Hello, world!' - # The target language - - result = [] - limit = 16 - for chunk in [messages[i:i + limit] for i in xrange(0, len(messages), limit)]: - # Translates some text into Russian - print("Requesting {} translations".format(len(chunk))) - translations = translate_client.translate(chunk, target_language=target_language) - - for translation in translations: - result.append({ - "source": translation["input"], - "translated": translation["translatedText"] - }) - return result - - -def translate_messages(source, destination, target_language): - with open(source) as f: - data = json.load(f) - - result = { - "translations": [], - "info": None - } - try: - with open(destination) as f: - result = json.load(f) - print("loaded old result") - except: - pass - - translations = result["translations"] - if translations is None: - print("Using new translation map") - translations = [] - else: - print("Loaded {} old translations".format(len(translations))) - - messages = [] - for message in data: - try: - messages.index(message["message"]) - except: - try: - found = False - for entry in translations: - if entry["key"]["message"] == message["message"]: - found = True - break - if not found: - raise Exception('add message for translate') - except: - messages.append(message["message"]) - - print("Translating {} messages".format(len(messages))) - if len(messages) != 0: - _translations = run_translate(messages, 'en', target_language) - print("Messages translated, generating target file") - - for translation in _translations: - translations.append({ - "key": { - "message": translation["source"] - }, - "translated": translation["translated"], - "flags": [ - "google-translate" - ] - }) - for translation in translations: - translation["translated"] = re.sub(r"% +([OoDdSs])", r" %\1", translation["translated"]) # Fix the broken "% o" or "% s" things - translation["translated"] = translation["translated"].replace("%O", "%o") # Replace all %O to %o - translation["translated"] = translation["translated"].replace("%S", "%s") # Replace all %S to %s - translation["translated"] = translation["translated"].replace("%D", "%d") # Replace all %D to %d - translation["translated"] = re.sub(r" +(%[ods])", r" \1", translation["translated"]) # Fix double spaces between a message and %s - translation["translated"] = re.sub(r"\( (%[ods])", r"(\1", translation["translated"]) # Fix the leading space after a brace: ( %s) - - print("Writing target file") - result["translations"] = translations - if result["info"] is None: - result["info"] = { - "contributors": [ - { - "name": "Google Translate, via script by Markus Hadenfeldt", - "email": "gtr.i18n.client@teaspeak.de" - } - ], - "name": "Auto translated messages for language " + target_language - } - - with io.open(destination, 'w', encoding='utf8') as f: - f.write(json.dumps(result, indent=2, ensure_ascii=False)) - print("Done") - - -def main(target_language): - target_file = "i18n/{}_google_translate.translation".format(target_language) - - translate_messages("generated/messages_script.json", target_file, target_language) - translate_messages("generated/messages_template.json", target_file, target_language) - pass - - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Invalid argument count!") - print("Usage: ./generate_i18n_gtranslate.py ") - exit(1) - - main(sys.argv[1]) diff --git a/shared/generate_i18n_gtranslate.ts b/shared/generate_i18n_gtranslate.ts new file mode 100644 index 00000000..1455013e --- /dev/null +++ b/shared/generate_i18n_gtranslate.ts @@ -0,0 +1,144 @@ +import * as path from "path"; +import * as fs from "fs-extra"; +import {TranslationServiceClient} from "@google-cloud/translate"; + +const translation_project_id = "luminous-shadow-92008"; +const translation_location = "global"; +const translation_client = new TranslationServiceClient(); +async function run_translate(messages: string[], source_language: string, target_language: string) : Promise<(string | undefined)[]> { + let messages_left = messages.slice(0); + let result = []; + + while (messages_left.length > 0) { + const chunk_size = Math.min(messages_left.length, 128); + const chunk = messages_left.slice(0, chunk_size); + + try { + const [response] = await translation_client.translateText({ + parent: `projects/${translation_project_id}/locations/${translation_location}`, + contents: chunk, + mimeType: "text/plain", + sourceLanguageCode: source_language, + targetLanguageCode: target_language + }); + + result.push(...response.translations.map(e => e.translatedText)); + } catch (error) { + console.log(error); + console.log("Failed to execute translation request: %o", 'details' in error ? error.details : error instanceof Error ? error.message : error); + throw "translated failed"; + } + + messages_left = messages.slice(chunk_size); + } + return result; +} + +interface TranslationFile { + info: { + name: string, + contributors: { + name: string, + email: string + }[] + }, + translations: { + translated: string, + flags: string[], + key: { + message: string + } + }[] +} + +interface InputFile { + message: string; + line: number; + character: number; + filename: string; +} + +async function translate_messages(input_file: string, output_file: string, source_language: string, target_language: string) { + let output_data: TranslationFile; + if(await fs.pathExists(output_file)) { + try { + output_data = await fs.readJSON(output_file); + } catch (error) { + console.log("Failed to parse output data: %o", error); + throw "failed to read output file"; + } + } else { + output_data = {} as any; + } + + if(!output_data.info) { + output_data.info = { + contributors: [ + { + "name": "Google Translate, via script by Markus Hadenfeldt", + "email": "gtr.i18n.client@teaspeak.de" + } + ], + name: "Auto translated messages for language " + target_language + } + } + + if(!Array.isArray(output_data.translations)) + output_data.translations = []; + + let messages_to_translate: InputFile[] = []; + try { + messages_to_translate = await fs.readJSON(input_file); + } catch (error) { + console.log("Failed to parse input file %o", error); + throw "failed to read input file"; + } + + const original_messages = messages_to_translate.length; + messages_to_translate = messages_to_translate.filter(e => output_data.translations.findIndex(f => e.message === f.key.message) === -1); + console.log("Messages to translate: %d out of %d", messages_to_translate.length, original_messages); + + const response = await run_translate(messages_to_translate.map(e => e.message), source_language, target_language); + if(messages_to_translate.length !== response.length) + throw "invalid response length"; + + for(let index = 0; index < response.length; index++) { + if(typeof response[index] !== "string") { + console.log("Failed to translate message %s", messages_to_translate[index]); + continue; + } + + output_data.translations.push({ + key: { + message: messages_to_translate[index].message + }, + translated: response[index], + flags: [ + "google-translated" + ] + }); + } + + await fs.writeJSON(output_file, output_data); +} + +const process_args = process.argv.slice(2); +if(process_args.length !== 1) { + console.error("Invalid argument count"); + console.error("Usage: ./generate_i18n_gtranslate.py "); + process.exit(1); +} + +const input_files = ["../dist/translations.json", "generated/translations_html.json"].map(e => path.join(__dirname, e)); +const output_file = path.join(__dirname, "i18n", process_args[0] + "_google_translate.translation"); + +(async () => { + for(const file of input_files) + await translate_messages(file, output_file, "en", process_args[0]); +})().catch(error => { + console.error("Failed to create translation files: %o", error); + process.exit(1); +}).then(() => { + console.log("Translation files have been updated."); + process.exit(0); +}); \ No newline at end of file diff --git a/shared/generate_translations.sh b/shared/generate_translations.sh index 1319e667..3e671cff 100644 --- a/shared/generate_translations.sh +++ b/shared/generate_translations.sh @@ -1,17 +1,9 @@ #!/usr/bin/env bash -BASEDIR=$(dirname "$0") -cd "$BASEDIR" +cd "$(dirname "$0")" || { echo "Failed to enter base directory"; exit 1; } -#Generate the script translations -npm run ttsc -- -p $(pwd)/tsconfig/tsconfig.json -if [ $? -ne 0 ]; then - echo "Failed to generate translation file for the script files" - exit 1 -fi - -npm run trgen -- -f $(pwd)/html/templates.html -d $(pwd)/generated/messages_template.json -if [ $? -ne 0 ]; then +npm run trgen -- -f "$(pwd)/html/templates.html" -f "$(pwd)/html/templates/modal/newcomer.html" -f "$(pwd)/html/templates/modal/musicmanage.html" -d "$(pwd)/generated/translations_html.json"; _exit_code=$? +if [[ $_exit_code -ne 0 ]]; then echo "Failed to generate translations file for the template files" exit 1 fi \ No newline at end of file