diff --git a/.github/workflows/check_for_crowdin_updates.yml b/.github/workflows/check_for_crowdin_updates.yml index 42a6430..873080c 100644 --- a/.github/workflows/check_for_crowdin_updates.yml +++ b/.github/workflows/check_for_crowdin_updates.yml @@ -169,8 +169,8 @@ jobs: - build_desktop: - name: Build Desktop strings + build_localization_module: + name: Build localization module strings needs: [parse_translations] runs-on: ubuntu-latest steps: @@ -182,76 +182,27 @@ jobs: - name: Setup shared uses: ./scripts/actions/setup_shared - - name: Checkout Desktop - uses: ./scripts/actions/checkout_desktop - - name: Download parsed translations uses: actions/download-artifact@v4 with: name: session-parsed path: "${{ github.workspace }}" - - name: Prepare Desktop Strings + - name: Generate TypeScript run: | - python "${{ github.workspace }}/scripts/crowdin/generate_desktop_strings.py" \ + python "${{ github.workspace }}/scripts/crowdin/codegen_localization.py" \ "${{ github.workspace }}/parsed_translations.json" \ - "${{ github.workspace }}/desktop/_locales" \ - "${{ github.workspace }}/desktop/ts/localization/constants.ts" - - - name: Validate strings for Desktop - run: cd ${{ github.workspace }}/desktop && npm install -g yarn && yarn build:locales + "${{ github.workspace }}/output/generated" - - name: Upload Desktop artifacts + - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: session-desktop + name: session-localization path: | - ${{ github.workspace }}/desktop/_locales - ${{ github.workspace }}/desktop/ts/localization/constants.ts - overwrite: true - if-no-files-found: warn - retention-days: 7 - - build_qa: - name: Build QA strings - needs: [parse_translations] - runs-on: ubuntu-latest - steps: - - name: Checkout Repo Content - uses: actions/checkout@v4 - with: - path: 'scripts' - # don't provide a branch (ref) so it uses the default for that event - - name: Setup shared - uses: ./scripts/actions/setup_shared - - - name: Checkout Desktop - uses: ./scripts/actions/checkout_desktop - - - name: Download parsed translations - uses: actions/download-artifact@v4 - with: - name: session-parsed - path: "${{ github.workspace }}" - - - name: Export QA Strings (json) - run: | - python "${{ github.workspace }}/scripts/crowdin/generate_desktop_strings.py" --qa_build \ - "${{ github.workspace }}/parsed_translations.json" \ - "${{ github.workspace }}/desktop/_locales" \ - "${{ github.workspace }}/desktop/ts/localization/constants.ts" - - name: Prepare QA strings (ts) - run: | - cd ${{ github.workspace }}/desktop/ - python ./tools/localization/generateLocales.py --generate-types --print-problems --print-problem-strings --print-problem-formatting-tag-strings --error-on-problems - cd - - - name: Upload QA artefacts - uses: actions/upload-artifact@v4 - with: - name: session-qa - path: | - ${{ github.workspace }}/desktop/ts/localization/locales.ts - ${{ github.workspace }}/desktop/ts/localization/constants.ts + ${{ github.workspace }}/output/generated/locales.ts + ${{ github.workspace }}/output/generated/english.ts + ${{ github.workspace }}/output/generated/translations.ts + ${{ github.workspace }}/output/generated/constants.ts overwrite: true if-no-files-found: warn retention-days: 7 @@ -325,7 +276,7 @@ jobs: jobs_sync: name: Waiting for build jobs - needs: [build_android, build_ios, build_desktop, build_qa] + needs: [build_android, build_ios, build_localization_module] runs-on: ubuntu-latest steps: @@ -364,9 +315,9 @@ jobs: commit-message: ${{ env.PR_TITLE }} delete-branch: true - make_desktop_pr: + make_localization_module_pr: needs: [jobs_sync] - name: Make Desktop PR + name: Make Localization Module PR runs-on: ubuntu-latest if: ${{ github.event_name == 'schedule' || inputs.UPDATE_PULL_REQUESTS == true }} steps: @@ -375,19 +326,19 @@ jobs: with: path: 'scripts' # don't provide a branch (ref) so it uses the default for that event - - name: Checkout Desktop - uses: ./scripts/actions/checkout_desktop + - name: Checkout Localization Module + uses: ./scripts/actions/checkout_localization_module - uses: actions/download-artifact@v4 with: - name: session-desktop - # this has to be the first shared parent on the upload artefact task for Desktop - path: "${{ github.workspace }}/desktop" + name: session-localization + # Download to /generated folder in localization module repo + path: "${{ github.workspace }}/module/generated" - - name: Create Desktop Pull Request + - name: Create Localization Module Pull Request uses: peter-evans/create-pull-request@v6 with: - path: 'desktop' + path: 'module' token: ${{ secrets.CROWDIN_PR_TOKEN }} title: ${{ env.PR_TITLE }} body: ${{ env.PR_DESCRIPTION }} @@ -427,32 +378,3 @@ jobs: commit-message: ${{ env.PR_TITLE }} delete-branch: true - make_qa_pr: - needs: [jobs_sync] - name: Make QA PR (Appium) - runs-on: ubuntu-latest - if: ${{ github.event_name == 'schedule' || inputs.UPDATE_PULL_REQUESTS == true }} - steps: - - name: Checkout Repo Content - uses: actions/checkout@v4 - with: - path: 'scripts' - - - name: Checkout Session Appium - uses: ./scripts/actions/checkout_qa - - - uses: actions/download-artifact@v4 - with: - name: session-qa - path: "${{ github.workspace }}/appium/run/localizer" - - - name: Create QA Pull Request - uses: peter-evans/create-pull-request@v6 - with: - path: 'appium' - token: ${{ secrets.CROWDIN_PR_TOKEN }} - title: ${{ env.PR_TITLE }} - body: ${{ env.PR_DESCRIPTION }} - branch: ${{ env.PR_TARGET_BRANCH }} - commit-message: ${{ env.PR_TITLE }} - delete-branch: true diff --git a/README.md b/README.md index 88cf0ee..60000c3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repo houses scripts which are shared between the different platform repos f ## Crowdin Translation Workflow -Automated workflow that downloads translations from Crowdin, validates them, and creates PRs for iOS, Android, and Desktop platforms. +Automated workflow that downloads translations from Crowdin, validates them, and creates PRs for iOS and Android platforms and for the Typescript Localization Module for Desktop and QA. ### Required Secrets diff --git a/actions/checkout_desktop/action.yml b/actions/checkout_desktop/action.yml deleted file mode 100644 index dc28d7f..0000000 --- a/actions/checkout_desktop/action.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: 'Setup for all' -description: "Setup shared for all jobs" -runs: - using: 'composite' - steps: - - name: Checkout Desktop - uses: actions/checkout@v4 - with: - repository: 'session-foundation/session-desktop' - path: 'desktop' - ref: 'dev' - - uses: actions/setup-node@v4 - with: - node-version-file: 'desktop/.nvmrc' - - - name: Remove existing strings - shell: bash - run: | - rm -rf ${{ github.workspace }}/desktop/_locales/* \ No newline at end of file diff --git a/actions/checkout_localization_module/action.yml b/actions/checkout_localization_module/action.yml new file mode 100644 index 0000000..f2bf527 --- /dev/null +++ b/actions/checkout_localization_module/action.yml @@ -0,0 +1,16 @@ +name: 'Setup for all' +description: "Setup shared for all jobs" +runs: + using: 'composite' + steps: + - name: Checkout Localization Module + uses: actions/checkout@v4 + with: + repository: 'session-foundation/session-localization' + path: 'module' + ref: 'main' + + - name: Remove existing strings + shell: bash + run: | + rm -rf ${{ github.workspace }}/generated/* diff --git a/actions/checkout_qa/action.yml b/actions/checkout_qa/action.yml deleted file mode 100644 index ff549f0..0000000 --- a/actions/checkout_qa/action.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: 'Checkout QA' -description: 'Checks out the Session Appium repository' -runs: - using: 'composite' - steps: - - name: Checkout Session Appium - uses: actions/checkout@v4 - with: - repository: session-foundation/session-appium - path: 'appium' - ref: 'main' \ No newline at end of file diff --git a/crowdin/codegen_localization.py b/crowdin/codegen_localization.py new file mode 100644 index 0000000..8ce7b43 --- /dev/null +++ b/crowdin/codegen_localization.py @@ -0,0 +1,668 @@ +#!/usr/bin/env python3 +""" +Localization Module Typescript Code Generator + +Generates TypeScript files for Session Desktop localization `ts/localization/generated/`: +- locales.ts: Types, utility functions, and re-exports +- constants.ts: Localization constants. +- english.ts: English strings (editable by devs for local work) +- translations.ts: Sparse translations for other locales (only actual translations, no duplicated English) +""" + +import argparse +import re +import os +from typing import Dict, List, Any, Tuple +from generate_shared import ( + load_parsed_translations, + print_progress, + print_success, + run_main +) + +# Mapping of variable names to their TypeScript "With*" type names +WITH_MAP = { + ("name", "string"): "WithName", + ("group_name", "string"): "WithGroupName", + ("community_name", "string"): "WithCommunityName", + ("other_name", "string"): "WithOtherName", + ("author", "string"): "WithAuthor", + ("emoji", "string"): "WithEmoji", + ("emoji_name", "string"): "WithEmojiName", + ("admin_name", "string"): "WithAdminName", + ("time", "string"): "WithTime", + ("time_large", "string"): "WithTimeLarge", + ("time_small", "string"): "WithTimeSmall", + ("disappearing_messages_type", "string"): "WithDisappearingMessagesType", + ("conversation_name", "string"): "WithConversationName", + ("file_type", "string"): "WithFileType", + ("date", "string"): "WithDate", + ("date_time", "string"): "WithDateTime", + ("message_snippet", "string"): "WithMessageSnippet", + ("query", "string"): "WithQuery", + ("version", "string"): "WithVersion", + ("information", "string"): "WithInformation", + ("device", "string"): "WithDevice", + ("percent_loader", "string"): "WithPercentLoader", + ("message_count", "string"): "WithMessageCount", + ("conversation_count", "string"): "WithConversationCount", + ("found_count", "number"): "WithFoundCount", + ("hash", "string"): "WithHash", + ("url", "string"): "WithUrl", + ("account_id", "string"): "WithAccountId", + ("count", "number"): "WithCount", + ("service_node_id", "string"): "WithServiceNodeId", + ("limit", "string"): "WithLimit", + ("relative_time", "string"): "WithRelativeTime", + ("icon", "string"): "WithIcon", + ("storevariant", "string"): "WithStoreVariant", + ("min", "string"): "WithMin", + ("max", "string"): "WithMax", +} + +LOCALE_KEY_MAPPING = { + 'en-US': 'en', + 'kmr-TR': 'kmr', + 'hy-AM': 'hy-AM', + 'es-419': 'es-419', + 'pt-BR': 'pt-BR', + 'pt-PT': 'pt-PT', + 'zh-CN': 'zh-CN', + 'zh-TW': 'zh-TW', + 'sr-CS': 'sr-CS', + 'sr-SP': 'sr-SP' +} + +DISCLAIMER_GENERATED = """// Do not modify this file manually. This file was generated by a script +// at https://github.com/session-foundation/session-shared-scripts +// Translations can be changed at https://getsession.org/translate + +""" + +DISCLAIMER_ENGLISH = """// English strings. Other locales will fall back to these strings when +// translations are missing. + +""" + + +def get_locale_key(locale: str, two_letter_code: str) -> str: + return LOCALE_KEY_MAPPING.get(locale, LOCALE_KEY_MAPPING.get(two_letter_code, two_letter_code)) + + +def escape_str(value: str) -> str: + """Escapes chars that would break the .ts: newlines and quotes.""" + return value.replace("\n", "\\n").replace("\'", "\\\'") + + +def wrap_value(value: str) -> str: + """Wraps the given value in single quotes if it contains special characters.""" + if re.search(r"[^a-zA-Z0-9_]", value): + return f"'{value}'" + return value + + +def extract_vars(text: str, glossary_keys: List[str]) -> List[str]: + """Extract all {variable} placeholders from a string, excluding glossary variables.""" + vars = re.findall(r'\{(.*?)\}', text) + return [var for var in vars if var not in glossary_keys] + + +def vars_to_record_ts(variables: List[str]) -> List[List[str]]: + """Convert variable names to [name, type] pairs.""" + arr = [] + for var in variables: + var_type = 'number' if var in ('count', 'found_count') else 'string' + to_append = [var, var_type] + if to_append not in arr: + arr.append(to_append) + return arr + + +def format_tokens_with_named_args(token_args_dict: Dict[str, List[List[str]]]) -> str: + """Format token->args mapping as TypeScript type.""" + if not token_args_dict: + return "{}" + + result = [] + for token, args in token_args_dict.items(): + extras = [] + with_types = [] + + for arg_name, arg_type in args: + key = (arg_name, arg_type) + if key in WITH_MAP: + with_types.append(WITH_MAP[key]) + else: + extras.append(f"{arg_name}: {arg_type}") + + joined = " & ".join(with_types) + if extras: + extras_str = "{ " + ", ".join(extras) + " }" + joined = f"{joined} & {extras_str}" if joined else extras_str + + result.append(f" {token}: {joined}") + + return "{\n" + ",\n".join(result) + "\n}" + + +def generate_with_types() -> str: + """Generate the With* type definitions.""" + lines = [] + for (arg_name, arg_type), type_name in WITH_MAP.items(): + lines.append(f"type {type_name} = {{{arg_name}: {arg_type}}};") + return "\n".join(lines) + + +def replace_glossary_variables(text: str, glossary_dict: Dict[str, str]) -> str: + """Replace glossary variables like {app_name} with their actual values.""" + for glossary_key, glossary_value in glossary_dict.items(): + text = text.replace("{" + glossary_key + "}", glossary_value) + return text + + +def snake_to_camel(snake_str: str) -> str: + parts = snake_str.split('_') + return parts[0].lower() + ''.join(word.capitalize() for word in parts[1:]) + + +def convert_parsed_to_flat_locales(parsed_data: Dict[str, Any], is_qa_build: bool = False) -> Dict[str, Dict[str, str]]: + """ + Convert parsed translation data to the flat locale format. + Returns: { "en": {"key1": "value1", ...}, "de": {...}, ... } + """ + source_language = parsed_data['source_language'] + target_languages = parsed_data['target_languages'] + locales_data = parsed_data['locales'] + glossary_dict = parsed_data.get('glossary', {}) + + languages_to_process = [source_language] + \ + ([] if is_qa_build else target_languages) + + result = {} + + for lang in languages_to_process: + orig_locale = lang['locale'] + locale_key = get_locale_key(orig_locale, lang['twoLettersCode']) + + locale_data = locales_data.get(orig_locale, {}) + translations = locale_data.get('translations', {}) + + flat_translations = {} + + for key, trans_data in translations.items(): + if trans_data['type'] == 'plural': + forms = trans_data['forms'] + parts = [] + for form, value in forms.items(): + if form in ['zero', 'one', 'two', 'few', 'many', 'other']: + parts.append(f"{form} [{value}]") + flat_translations[key] = "{count, plural, " + \ + " ".join(parts) + "}" + else: + flat_translations[key] = trans_data['value'] + + # Add glossary items to English locale (converted to camelCase) + if locale_key == 'en': + for glossary_key, glossary_value in glossary_dict.items(): + camel_key = snake_to_camel(glossary_key) + flat_translations[camel_key] = glossary_value + + result[locale_key] = flat_translations + + return result + + +def categorize_strings( + en_locale: Dict[str, str], + glossary_dict: Dict[str, str] +) -> Tuple[List[str], Dict[str, List[List[str]]], Dict[str, List[List[str]]]]: + """ + Categorize English strings into: + - tokens_no_args: strings without dynamic variables + - tokens_simple_with_args: simple strings with variables + - tokens_plural_with_args: plural strings with variables + + Returns: + (tokens_no_args, tokens_simple_with_args, tokens_plural_with_args) + """ + glossary_keys = list(glossary_dict.keys()) + plural_pattern = r"(zero|one|two|few|many|other)\s*\[([^\]]+)\]" + + tokens_no_args = [] + tokens_simple_with_args = {} + tokens_plural_with_args = {} + + for key, value_en in sorted(en_locale.items()): + if value_en.startswith("{count, plural, "): + # Plural string + en_plurals_with_token = re.findall(plural_pattern, value_en) + if en_plurals_with_token: + extracted_vars = extract_vars( + en_plurals_with_token[0][1], glossary_keys) + if 'count' not in extracted_vars: + extracted_vars.append('count') + tokens_plural_with_args[key] = vars_to_record_ts( + extracted_vars) + else: + # Simple string + replaced_value_en = replace_glossary_variables( + value_en, glossary_dict) + extracted_vars = extract_vars(replaced_value_en, glossary_keys) + + if extracted_vars: + tokens_simple_with_args[key] = vars_to_record_ts( + extracted_vars) + else: + tokens_no_args.append(key) + + return tokens_no_args, tokens_simple_with_args, tokens_plural_with_args + + +def generate_english_dictionary( + en_locale: Dict[str, str], + glossary_dict: Dict[str, str], + keys: List[str] +) -> str: + """Generate a TypeScript dictionary for English strings.""" + lines = [] + for key in sorted(keys): + if key not in en_locale: + continue + value = en_locale[key] + # Replace glossary variables + value = replace_glossary_variables(value, glossary_dict) + escaped_value = escape_str(value) + lines.append(f" {wrap_value(key)}: \'{escaped_value}\'") + + return "{\n" + ",\n".join(lines) + ",\n}" + + +def generate_english_plural_dictionary( + en_locale: Dict[str, str], + glossary_dict: Dict[str, str], + keys: List[str] +) -> str: + plural_pattern = r"(zero|one|two|few|many|other)\s*\[([^\]]+)\]" + entries = [] + + for key in sorted(keys): + if key not in en_locale: + continue + value = en_locale[key] + plurals_with_token = re.findall(plural_pattern, value) + + form_lines = [] + for token, localized_string in plurals_with_token: + replaced_string = replace_glossary_variables( + localized_string, glossary_dict) + form_lines.append(f" {token}: \'{escape_str(replaced_string)}\'") + + entries.append(f" {wrap_value(key)}: {{\n" + ",\n".join(form_lines) + ",\n }") + + return "{\n" + ",\n".join(entries) + ",\n}" + + +def generate_sparse_translations( + locales: Dict[str, Dict[str, str]], + en_locale: Dict[str, str], + glossary_dict: Dict[str, str], + tokens_no_args: List[str], + tokens_simple_with_args: Dict[str, Any], + tokens_plural_with_args: Dict[str, Any] +) -> Tuple[str, str, str]: + """ + Generate sparse translation dictionaries for non-English locales. + Only includes strings that differ from English (actual translations). + + Returns: + (simple_no_args_dict, simple_with_args_dict, plurals_dict) + """ + plural_pattern = r"(zero|one|two|few|many|other)\s*\[([^\]]+)\]" + + # Get non-English locales + other_locales = {k: v for k, v in locales.items() if k != 'en'} + + # Build sparse dictionaries + sparse_no_args = {locale: {} for locale in other_locales} + sparse_with_args = {locale: {} for locale in other_locales} + sparse_plurals = {locale: {} for locale in other_locales} + + for locale, translations in other_locales.items(): + for key in tokens_no_args: + en_value = replace_glossary_variables( + en_locale.get(key, ""), glossary_dict) + locale_value = replace_glossary_variables( + translations.get(key, ""), glossary_dict) + + # Only include if different from English and not empty + if locale_value and locale_value != en_value: + sparse_no_args[locale][key] = locale_value + + for key in tokens_simple_with_args: + en_value = replace_glossary_variables( + en_locale.get(key, ""), glossary_dict) + locale_value = replace_glossary_variables( + translations.get(key, ""), glossary_dict) + + if locale_value and locale_value != en_value: + sparse_with_args[locale][key] = locale_value + + for key in tokens_plural_with_args: + en_value = en_locale.get(key, "") + locale_value = translations.get(key, "") + + # For plurals, compare the full ICU string + if locale_value and locale_value != en_value: + # Parse and store plural forms + plurals_with_token = re.findall(plural_pattern, locale_value) + if plurals_with_token: + forms = {} + for token, localized_string in plurals_with_token: + replaced = replace_glossary_variables( + localized_string, glossary_dict) + forms[token] = replaced + sparse_plurals[locale][key] = forms + + # Generate TypeScript dictionaries + def format_simple_sparse(sparse_dict: Dict[str, Dict[str, str]]) -> str: + locale_entries = [] + for locale in sorted(sparse_dict.keys()): + translations = sparse_dict[locale] + if not translations: + continue + string_entries = [] + for key in sorted(translations.keys()): + value = escape_str(translations[key]) + string_entries.append(f" {wrap_value(key)}: \'{value}\'") + locale_entries.append(f" {wrap_value(locale)}: {{\n" + ",\n".join(string_entries) + ",\n }") + if not locale_entries: + return "{}" + return "{\n" + ",\n".join(locale_entries) + ",\n}" + + def format_plural_sparse(sparse_dict: Dict[str, Dict[str, Dict[str, str]]]) -> str: + locale_entries = [] + for locale in sorted(sparse_dict.keys()): + key_forms = sparse_dict[locale] + if not key_forms: + continue + key_entries = [] + for key in sorted(key_forms.keys()): + forms = key_forms[key] + form_entries = [] + for form in ['zero', 'one', 'two', 'few', 'many', 'other']: + if form in forms: + form_entries.append(f" {form}: \'{escape_str(forms[form])}\'") + key_entries.append(f" {wrap_value(key)}: {{\n" + ",\n".join(form_entries) + ",\n }") + locale_entries.append(f" {wrap_value(locale)}: {{\n" + ",\n".join(key_entries) + ",\n }") + if not locale_entries: + return "{}" + return "{\n" + ",\n".join(locale_entries) + ",\n}" + + return ( + format_simple_sparse(sparse_no_args), + format_simple_sparse(sparse_with_args), + format_plural_sparse(sparse_plurals) + ) + + +def generate_english_ts( + en_locale: Dict[str, str], + glossary_dict: Dict[str, str], + tokens_no_args: List[str], + tokens_simple_with_args: Dict[str, Any], + tokens_plural_with_args: Dict[str, Any], + output_path: str +): + simple_no_args_dict = generate_english_dictionary( + en_locale, glossary_dict, tokens_no_args + ) + simple_with_args_dict = generate_english_dictionary( + en_locale, glossary_dict, list(tokens_simple_with_args.keys()) + ) + plural_dict = generate_english_plural_dictionary( + en_locale, glossary_dict, list(tokens_plural_with_args.keys()) + ) + + content = f"""{DISCLAIMER_GENERATED}{DISCLAIMER_ENGLISH}import type {{ TokenSimpleNoArgs, TokenSimpleWithArgs, TokenPluralWithArgs, PluralForms }} from './locales'; + +/** English strings without dynamic arguments */ +export const enSimpleNoArgs = {simple_no_args_dict} as const satisfies Record; + +/** English strings with dynamic arguments */ +export const enSimpleWithArgs = {simple_with_args_dict} as const satisfies Record; + +/** English plural strings */ +export const enPlurals = {plural_dict} as const satisfies Record; +""" + + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + with open(output_path, 'w', encoding='utf-8') as f: + f.write(content) + + print_success(f"Generated {output_path}") + + +def generate_translations_ts( + locales: Dict[str, Dict[str, str]], + en_locale: Dict[str, str], + glossary_dict: Dict[str, str], + tokens_no_args: List[str], + tokens_simple_with_args: Dict[str, Any], + tokens_plural_with_args: Dict[str, Any], + output_path: str +): + sparse_no_args, sparse_with_args, sparse_plurals = generate_sparse_translations( + locales, en_locale, glossary_dict, + tokens_no_args, tokens_simple_with_args, tokens_plural_with_args + ) + + # Get list of non-English locales + other_locales = sorted([k for k in locales.keys() if k != 'en']) + other_locales_type = " | ".join(f"'{locale}'" for locale in other_locales) + + content = f"""{DISCLAIMER_GENERATED}import type {{ TokenSimpleNoArgs, TokenSimpleWithArgs, TokenPluralWithArgs, PluralForms }} from './locales'; + +/** Non-English locale codes */ +export type TranslationLocale = {other_locales_type}; + +/** Sparse translations for simple strings without arguments (only actual translations, no English duplicates) */ +export const translationsSimpleNoArgs: Partial>>> = {sparse_no_args} as const; + +/** Sparse translations for simple strings with arguments */ +export const translationsSimpleWithArgs: Partial>>> = {sparse_with_args} as const; + +/** Sparse translations for plural strings */ +export const translationsPlurals: Partial>>> = {sparse_plurals} as const; +""" + + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + with open(output_path, 'w', encoding='utf-8') as f: + f.write(content) + + print_success(f"Generated {output_path}") + + +def generate_locales_ts( + tokens_no_args: List[str], + tokens_simple_with_args: Dict[str, Any], + tokens_plural_with_args: Dict[str, Any], + locales: Dict[str, Dict[str, str]], + rtl_languages: List[Dict], + glossary_dict: Dict[str, str], + output_path: str +): + """Generate locales.ts - types, and utility functions.""" + # Token type strings + tokens_no_args_str = "\n '" + \ + "' |\n '".join(tokens_no_args) + "'" if tokens_no_args else "never" + tokens_simple_with_args_str = "\n '" + "' |\n '".join(list( + tokens_simple_with_args.keys())) + "'" if tokens_simple_with_args else "never" + tokens_plural_with_args_str = "\n '" + "' |\n '".join(list( + tokens_plural_with_args.keys())) + "'" if tokens_plural_with_args else "never" + + tokens_union_simple_args = format_tokens_with_named_args( + tokens_simple_with_args) + tokens_union_plural_args = format_tokens_with_named_args( + tokens_plural_with_args) + + content = f"""{DISCLAIMER_GENERATED}// Re-export English strings and translations +export {{ enSimpleNoArgs, enSimpleWithArgs, enPlurals }} from './english'; +export {{ translationsSimpleNoArgs, translationsSimpleWithArgs, translationsPlurals, type TranslationLocale }} from './translations'; + +// ============================================================================ +// Type Definitions +// ============================================================================ + +{generate_with_types()} + +/** Plural form keys */ +export type PluralForm = 'zero' | 'one' | 'two' | 'few' | 'many' | 'other'; + +/** Plural forms object */ +export type PluralForms = Partial>; + +/** Token keys for simple strings without arguments */ +export type TokenSimpleNoArgs = {tokens_no_args_str}; + +/** Token keys for simple strings with arguments */ +export type TokenSimpleWithArgs = {tokens_simple_with_args_str}; + +/** Token keys for plural strings */ +export type TokenPluralWithArgs = {tokens_plural_with_args_str}; + +/** Argument types for simple strings with arguments */ +export type TokensSimpleAndArgs = {tokens_union_simple_args}; + +/** Argument types for plural strings */ +export type TokensPluralAndArgs = {tokens_union_plural_args}; +""" + + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + with open(output_path, 'w', encoding='utf-8') as f: + f.write(content) + + print_success(f"Generated {output_path}") + + +def generate_constants_ts( + locales: Dict[str, Dict[str, str]], + rtl_languages: List[Dict], + output_path: str +): + all_locales = sorted(locales.keys()) + rtl_locales = sorted([lang["twoLettersCode"] for lang in rtl_languages]) + + crowdin_locales_str = ",".join(f"\n '{locale}'" for locale in all_locales) + rtl_locales_str = ", ".join(f"'{locale}'" for locale in rtl_locales) + + content = f"""{DISCLAIMER_GENERATED} +/** Right-to-left locale codes */ +export const rtlLocales = [{rtl_locales_str}]; + +/** All supported Crowdin locale codes */ +export const crowdinLocales = [{crowdin_locales_str}, +] as const; + +/** Crowdin locale type */ +export type CrowdinLocale = (typeof crowdinLocales)[number]; + +/** Type guard for CrowdinLocale */ +export function isCrowdinLocale(locale: string): locale is CrowdinLocale {{ + return crowdinLocales.indexOf(locale as CrowdinLocale) !== -1; +}} +""" + + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + with open(output_path, 'w', encoding='utf-8') as f: + f.write(content) + + print_success(f"Generated {output_path}") + + +def main(): + parser = argparse.ArgumentParser( + description='Generate Desktop localization files from parsed translations' + ) + parser.add_argument( + 'parsed_translations_file', + help='Path to the parsed translations JSON file' + ) + parser.add_argument( + 'output_directory', + help='Directory to write the output files (locales.ts, english.ts, translations.ts)' + ) + parser.add_argument( + '--qa-build', + action='store_true', + help='Generate only English translations (for QA builds)' + ) + args = parser.parse_args() + + # Load parsed data + print_progress("Loading parsed translations...") + parsed_data = load_parsed_translations(args.parsed_translations_file) + glossary_dict = parsed_data.get('glossary', {}) + rtl_languages = parsed_data.get('rtl_languages', []) + + # Convert to flat locale format + print_progress("Converting parsed translations to locale format...") + locales = convert_parsed_to_flat_locales(parsed_data, args.qa_build) + en_locale = locales.get('en', {}) + + # Categorize strings + print_progress("Categorizing strings...") + tokens_no_args, tokens_simple_with_args, tokens_plural_with_args = categorize_strings( + en_locale, glossary_dict + ) + + # Calculate statistics + total_tokens = len(tokens_no_args) + \ + len(tokens_simple_with_args) + len(tokens_plural_with_args) + print_success(f"Found {total_tokens} total tokens:") + print_success(f" - {len(tokens_no_args)} simple without args") + print_success(f" - {len(tokens_simple_with_args)} simple with args") + print_success(f" - {len(tokens_plural_with_args)} plurals") + + # Generate output files + output_dir = args.output_directory + os.makedirs(output_dir, exist_ok=True) + + # Generate english.ts + print_progress("Generating english.ts...") + generate_english_ts( + en_locale, glossary_dict, + tokens_no_args, tokens_simple_with_args, tokens_plural_with_args, + os.path.join(output_dir, 'english.ts') + ) + + if args.qa_build: + print_progress("Skipping translations.ts as this is a qa build...") + else: + # Generate translations.ts + print_progress("Generating translations.ts...") + generate_translations_ts( + locales, en_locale, glossary_dict, + tokens_no_args, tokens_simple_with_args, tokens_plural_with_args, + os.path.join(output_dir, 'translations.ts') + ) + + # Generate locales.ts + print_progress("Generating locales.ts...") + generate_locales_ts( + tokens_no_args, tokens_simple_with_args, tokens_plural_with_args, + locales, rtl_languages, glossary_dict, + os.path.join(output_dir, 'locales.ts') + ) + + # Generate constants.ts (for backwards compatibility) + print_progress("Generating constants.ts...") + generate_constants_ts( + locales, rtl_languages, + os.path.join(output_dir, 'constants.ts') + ) + + print_success("All files generated successfully!") + + +if __name__ == "__main__": + run_main(main) diff --git a/crowdin/generate_desktop_strings.py b/crowdin/generate_desktop_strings.py deleted file mode 100644 index 6e2ec38..0000000 --- a/crowdin/generate_desktop_strings.py +++ /dev/null @@ -1,208 +0,0 @@ -import os -import json -import re -import argparse -from pathlib import Path -from typing import Dict, List, Any -from generate_shared import ( - load_parsed_translations, - clean_string, - print_progress, - print_success, - run_main -) - -# Customizable mapping for output folder hierarchy -# Add entries here to customize the output path for specific locales -# Format: 'input_locale': 'output_path' -LOCALE_PATH_MAPPING = { - 'en-US': 'en', - 'kmr-TR': 'kmr', - # Note: we don't want to replace - with _ anymore. - # We still need those mappings, otherwise they fallback to their 2 letter codes - 'hy-AM': 'hy-AM', - 'es-419': 'es-419', - 'pt-BR': 'pt-BR', - 'pt-PT': 'pt-PT', - 'zh-CN': 'zh-CN', - 'zh-TW': 'zh-TW', - 'sr-CS': 'sr-CS', - 'sr-SP': 'sr-SP' - # Add more mappings as needed -} - - -def matches_braced_pattern(string: str) -> bool: - return re.search(r"\{(.+?)\}", string) is not None - - -def snake_to_camel(snake_str: str) -> str: - parts = snake_str.split('_') - return parts[0].lower() + ''.join(word.capitalize() for word in parts[1:]) - - -def generate_icu_pattern(trans_data: Dict[str, Any] | str, glossary_dict: Dict[str, str]) -> str: - """ - Generate an ICU pattern from translation data. - Args: - trans_data: Either a dict with 'type' and 'forms'/'value', or a raw string - glossary_dict: Dictionary of non-translatable strings - Returns: - ICU-formatted string - """ - # Handle raw strings (from glossary) - if isinstance(trans_data, str): - return clean_string(trans_data, False, glossary_dict, {}) - if trans_data['type'] == 'plural': - pattern_parts = [] - for form, value in trans_data['forms'].items(): - if form in ['zero', 'one', 'two', 'few', 'many', 'other', 'exact', 'fractional']: - cleaned_value = clean_string(value, False, glossary_dict, {}) - pattern_parts.append(f"{form} [{cleaned_value}]") - return "{{count, plural, {0}}}".format(" ".join(pattern_parts)) - else: - return clean_string(trans_data['value'], False, glossary_dict, {}) - - -def get_output_locale(locale: str, two_letter_code: str) -> str: - return LOCALE_PATH_MAPPING.get(locale, LOCALE_PATH_MAPPING.get(two_letter_code, two_letter_code)) - - -def convert_locale_to_json( - translations: Dict[str, Any], - glossary_dict: Dict[str, str], - output_dir: str, - locale: str, - two_letter_code: str -) -> str: - """ - Convert translations for a single locale to JSON format. - Args: - translations: Dictionary of translations for this locale - glossary_dict: Dictionary of non-translatable strings - output_dir: Base output directory - locale: Full locale code - two_letter_code: Two-letter language code - Returns: - The output locale name used - """ - sorted_translations = sorted(translations.items()) - converted_translations = {} - - for resname, trans_data in sorted_translations: - converted_translations[resname] = generate_icu_pattern(trans_data, glossary_dict) - - # Add glossary items (converted to camelCase) - for resname, text in glossary_dict.items(): - converted_translations[snake_to_camel(resname)] = generate_icu_pattern(text, glossary_dict) - - # Write output - output_locale = get_output_locale(locale, two_letter_code) - locale_output_dir = os.path.join(output_dir, output_locale) - output_file = os.path.join(locale_output_dir, 'messages.json') - os.makedirs(locale_output_dir, exist_ok=True) - - with open(output_file, 'w', encoding='utf-8') as file: - json.dump(converted_translations, file, ensure_ascii=False, indent=2, sort_keys=True) - file.write('\n\n') - - return output_locale - - -def generate_typescript_constants( - glossary_dict: Dict[str, str], - output_path: str, - exported_locales: List[str], - rtl_languages: List[Dict] -): - rtl_locales = sorted([lang["twoLettersCode"] for lang in rtl_languages]) - - Path(output_path).parent.mkdir(parents=True, exist_ok=True) - - joined_exported_locales = ",".join(f"\n '{locale}'" for locale in exported_locales) - joined_rtl_locales = ", ".join(f"'{locale}'" for locale in rtl_locales) - - with open(output_path, 'w', encoding='utf-8') as file: - file.write('export enum LOCALE_DEFAULTS {\n') - - for key, text in glossary_dict.items(): - if not matches_braced_pattern(text): - cleaned_text = clean_string(text, False, glossary_dict, {}) - file.write(f" {key} = '{cleaned_text}',\n") - - file.write('}\n\n') - file.write(f"export const rtlLocales = [{joined_rtl_locales}];\n\n") - file.write(f"export const crowdinLocales = [{joined_exported_locales},\n] as const;\n\n") - file.write("export type CrowdinLocale = (typeof crowdinLocales)[number];\n\n") - file.write('export function isCrowdinLocale(locale: string): locale is CrowdinLocale {\n') - file.write(' return crowdinLocales.includes(locale as CrowdinLocale);\n') - file.write('}\n\n') - - -def main(): - parser = argparse.ArgumentParser(description='Convert parsed translations to JSON.') - parser.add_argument( - '--qa_build', - help='Set to true to output only English strings (only used for QA)', - action=argparse.BooleanOptionalAction - ) - parser.add_argument( - 'parsed_translations_file', - help='Path to the parsed translations JSON file' - ) - parser.add_argument( - 'translations_output_directory', - help='Directory to save the converted translation files' - ) - parser.add_argument( - 'non_translatable_strings_output_path', - help='Path to save the non-translatable strings to' - ) - args = parser.parse_args() - is_qa_build = args.qa_build or False - - parsed_data = load_parsed_translations(args.parsed_translations_file) - glossary_dict = parsed_data['glossary'] - source_language = parsed_data['source_language'] - target_languages = parsed_data['target_languages'] - rtl_languages = parsed_data['rtl_languages'] - locales = parsed_data['locales'] - - print_progress("Converting translations to target format...") - exported_locales = [] - - languages_to_process = [source_language] + ([] if is_qa_build else target_languages) - - for language in languages_to_process: - lang_locale = language['locale'] - lang_two_letter_code = language['twoLettersCode'] - - print_progress(f"Converting translations for {lang_locale} to target format...") - - locale_data = locales.get(lang_locale) - if locale_data is None: - raise ValueError(f"Missing locale data for {lang_locale}") - - exported_as = convert_locale_to_json( - locale_data['translations'], - glossary_dict, - args.translations_output_directory, - lang_locale, - lang_two_letter_code - ) - exported_locales.append(exported_as) - - print_success("All conversions complete") - - print_progress("Generating static strings file...") - generate_typescript_constants( - glossary_dict, - args.non_translatable_strings_output_path, - exported_locales, - rtl_languages - ) - print_success("Static string generation complete") - - -if __name__ == "__main__": - run_main(main)