Merge pull request #1109 from DisasterMo/crowdin-sync

Add automatic Crowdin synchronization

.github/workflows/crowdin_prep.yml

+# Prepare source texts & upload them to Crowdin
+
+name: Crowdin Source Texts Upload
+
+# on change to the English texts
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'core/libretro/libretro_core_options.h'
+
+jobs:
+  upload_source_file:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Java JDK
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Upload Source
+        shell: bash
+        env:
+          CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
+        run: |
+          python3 intl/download_workflow.py $CROWDIN_API_KEY "flycast" "core/libretro/libretro_core_options.h"

.github/workflows/crowdin_translate.yml

+# Download translations form Crowdin & Recreate libretro_core_options_intl.h
+
+name: Crowdin Translation Integration
+
+on:
+  schedule:
+  # please choose a random time to avoid all repos synching at the same time
+    - cron: '35 19 * * *'  # 11:35 AM, UTC
+
+jobs:
+  create_intl_file:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Java JDK
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token.
+          fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
+
+      - name: Create intl file
+        shell: bash
+        env:
+          CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
+        run: |
+          python3 intl/download_workflow.py $CROWDIN_API_KEY "flycast" "core/libretro/libretro_core_options_intl.h"
+
+      - name: Commit files
+        run: |
+          git config --local user.email "github-actions@github.com"
+          git config --local user.name "github-actions[bot]"
+          git add intl/download_workflow.py "core/libretro/libretro_core_options_intl.h"
+          git commit -m "Fetch translations & Recreate libretro_core_options_intl.h"
+
+      - name: GitHub Push
+        uses: ad-m/github-push-action@v0.6.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ github.ref }}

core/libretro/libretro.cpp

@@ -114,6 +114,8 @@ static bool allow_service_buttons = false;
 
 static bool libretro_supports_bitmasks = false;
 
+static bool categoriesSupported = false;
+
 u32 kcode[4] = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF};
 u8 rt[4] = {0, 0, 0, 0};
 u8 lt[4] = {0, 0, 0, 0};
@@ -285,7 +287,7 @@ void retro_set_environment(retro_environment_t cb)
 {
    environ_cb = cb;
 
-   libretro_set_core_options(environ_cb);
+   libretro_set_core_options(environ_cb, &categoriesSupported);
 
    static const struct retro_controller_description ports_default[] =
    {
@@ -398,6 +400,10 @@ static void set_variable_visibility(void)
    environ_cb(RETRO_ENVIRONMENT_SET_CORE_OPTIONS_DISPLAY, &option_display);
    option_display.key = CORE_OPTION_NAME "_per_content_vmus";
    environ_cb(RETRO_ENVIRONMENT_SET_CORE_OPTIONS_DISPLAY, &option_display);
+
+   /* only show, if categories not supported */
+   option_display.visible = ((settings.System == DC_PLATFORM_DREAMCAST) &&
+                             (!categoriesSupported));
    option_display.key = CORE_OPTION_NAME "_show_vmu_screen_settings";
    environ_cb(RETRO_ENVIRONMENT_SET_CORE_OPTIONS_DISPLAY, &option_display);
 
@@ -436,7 +442,9 @@ static void set_variable_visibility(void)
       option_display.visible = true;
       var.key = CORE_OPTION_NAME "_show_vmu_screen_settings";
 
-      if (environ_cb(RETRO_ENVIRONMENT_GET_VARIABLE, &var) && var.value)
+      if (environ_cb(RETRO_ENVIRONMENT_GET_VARIABLE, &var)
+          && var.value
+          && !categoriesSupported)
          if (!strcmp(var.value, "disabled"))
             option_display.visible = false;
    }
@@ -467,11 +475,18 @@ static void set_variable_visibility(void)
       environ_cb(RETRO_ENVIRONMENT_SET_CORE_OPTIONS_DISPLAY, &option_display);
    }
 
+   /* only show, if categories not supported */
+   option_display.visible = !categoriesSupported;
+   option_display.key = CORE_OPTION_NAME "_show_lightgun_settings";
+   environ_cb(RETRO_ENVIRONMENT_SET_CORE_OPTIONS_DISPLAY, &option_display);
+
    /* Show/hide light gun options */
    option_display.visible = true;
    var.key = CORE_OPTION_NAME "_show_lightgun_settings";
 
-   if (environ_cb(RETRO_ENVIRONMENT_GET_VARIABLE, &var) && var.value)
+   if (environ_cb(RETRO_ENVIRONMENT_GET_VARIABLE, &var)
+       && var.value
+       && !categoriesSupported)
       if (!strcmp(var.value, "disabled"))
          option_display.visible = false;
 

intl/.gitignore

+__pycache__
+crowdin-cli.jar
+*.h
+*.json

intl/core_option_regex.py

+import re
+
+# 0: full struct; 1: up to & including first []; 2: content between first {}
+p_struct = re.compile(r'(struct\s*[a-zA-Z0-9_\s]+\[])\s*'
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+)\s*)*'
+                      r'=\s*'  # =
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+)\s*)*'
+                      r'{((?:.|[\r\n])*?)\{\s*NULL,\s*NULL,\s*NULL\s*(?:.|[\r\n])*?},?(?:.|[\r\n])*?};')  # captures full struct, it's beginning and it's content
+# 0: type name[]; 1: type; 2: name
+p_type_name = re.compile(r'(retro_core_option_[a-zA-Z0-9_]+)\s*'
+                         r'(option_cats([a-z_]{0,8})|option_defs([a-z_]{0,8}))\s*\[]')
+# 0: full option; 1: key; 2: description; 3: additional info; 4: key/value pairs
+p_option = re.compile(r'{\s*'  # opening braces
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r'(\".*?\"|'  # key start; group 1
+                      r'[a-zA-Z0-9_]+\s*\((?:.|[\r\n])*?\)|'
+                      r'[a-zA-Z0-9_]+\s*\[(?:.|[\r\n])*?]|'
+                      r'[a-zA-Z0-9_]+\s*\".*?\")\s*'  # key end
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r',\s*'  # comma
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r'(\".*?\")\s*'  # description; group 2
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r',\s*'  # comma
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r'((?:'  # group 3
+                      r'(?:NULL|\"(?:.|[\r\n])*?\")\s*'  # description in category, info, info in category, category
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r',?\s*'  # comma
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r')+)'
+                      r'(?:'  # defs only start
+                      r'{\s*'  # opening braces
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r'((?:'  # key/value pairs start; group 4
+                      r'{\s*'  # opening braces
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r'(?:NULL|\".*?\")\s*'  # option key
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r',\s*'  # comma
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r'(?:NULL|\".*?\")\s*'  # option value
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r'}\s*'  # closing braces
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r',?\s*'  # comma
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r')*)'  # key/value pairs end
+                      r'}\s*'  # closing braces
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r',?\s*'  # comma
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r'(?:'  # defaults start
+                      r'(?:NULL|\".*?\")\s*'  # default value
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r',?\s*'  # comma
+                      r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                      r')*'  # defaults end
+                      r')?'  # defs only end
+                      r'},')  # closing braces
+# analyse option group 3
+p_info = re.compile(r'(NULL|\"(?:.|[\r\n])*?\")\s*'  # description in category, info, info in category, category
+                    r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                    r',')
+p_info_cat = re.compile(r'(NULL|\"(?:.|[\r\n])*?\")')
+# analyse option group 4
+p_key_value = re.compile(r'{\s*'  # opening braces
+                         r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                         r'(NULL|\".*?\")\s*'  # option key; 1
+                         r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                         r',\s*'  # comma
+                         r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                         r'(NULL|\".*?\")\s*'  # option value; 2
+                         r'(?:(?:\/\*(?:.|[\r\n])*?\*\/|\/\/.*[\r\n]+|#.*[\r\n]+)\s*)*'
+                         r'}')
+
+p_masked = re.compile(r'([A-Z_][A-Z0-9_]+)\s*(\"(?:"\s*"|\\\s*|.)*\")')
+
+p_intl = re.compile(r'(struct retro_core_option_definition \*option_defs_intl\[RETRO_LANGUAGE_LAST]) = {'
+                    r'((?:.|[\r\n])*?)};')
+p_set = re.compile(r'static INLINE void libretro_set_core_options\(retro_environment_t environ_cb\)'
+                   r'(?:.|[\r\n])*?};?\s*#ifdef __cplusplus\s*}\s*#endif')
+
+p_yaml = re.compile(r'"project_id": "[0-9]+".*\s*'
+                    r'"api_token": "([a-zA-Z0-9]+)".*\s*'
+                    r'"base_path": "\./intl".*\s*'
+                    r'"base_url": "https://api\.crowdin\.com".*\s*'
+                    r'"preserve_hierarchy": true.*\s*'
+                    r'"files": \[\s*'
+                    r'\{\s*'
+                    r'"source": "/_us/\*\.json",.*\s*'
+                    r'"translation": "/_%two_letters_code%/%original_file_name%",.*\s*'
+                    r'"skip_untranslated_strings": true.*\s*'
+                    r'},\s*'
+                    r']')

intl/crowdin.yaml

+"project_id": "380544"
+"api_token": "_secret_"
+"base_url": "https://api.crowdin.com"
+"preserve_hierarchy": true
+
+"files":
+  [
+    {
+      "source": "/intl/_core_name_/_us.json",
+      "dest": "/_core_name_/_core_name_.json",
+      "translation": "/intl/_core_name_/_%two_letters_code%.json",
+    },
+  ]

intl/crowdin_prep.py

+#!/usr/bin/env python3
+
+import core_option_translation as t
+
+
+if __name__ == '__main__':
+    try:
+        if t.os.path.isfile(t.sys.argv[1]):
+            _temp = t.os.path.dirname(t.sys.argv[1])
+        else:
+            _temp = t.sys.argv[1]
+        while _temp.endswith('/') or _temp.endswith('\\'):
+            _temp = _temp[:-1]
+        TARGET_DIR_PATH = _temp
+    except IndexError:
+        TARGET_DIR_PATH = t.os.path.dirname(t.os.path.dirname(t.os.path.realpath(__file__)))
+        print("No path provided, assuming parent directory:\n" + TARGET_DIR_PATH)
+
+    CORE_NAME = t.clean_file_name(t.sys.argv[2])
+    DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
+    H_FILE_PATH = t.os.path.join(TARGET_DIR_PATH, 'libretro_core_options.h')
+
+    print('Getting texts from libretro_core_options.h')
+    with open(H_FILE_PATH, 'r+', encoding='utf-8') as _h_file:
+        _main_text = _h_file.read()
+    _hash_n_str = t.get_texts(_main_text)
+    _files = t.create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str)
+
+    _source_jsons = t.h2json(_files)
+
+    print('\nAll done!')

intl/crowdin_source_upload.py

+#!/usr/bin/env python3
+
+import re
+import os
+import shutil
+import subprocess
+import sys
+import urllib.request
+import zipfile
+import core_option_translation as t
+
+# --------------------          MAIN          -------------------- #
+
+if __name__ == '__main__':
+    # Check Crowdin API Token and core name
+    try:
+        API_KEY = sys.argv[1]
+        CORE_NAME = t.clean_file_name(sys.argv[2])
+    except IndexError as e:
+        print('Please provide Crowdin API Token and core name!')
+        raise e
+
+    DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
+    YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml')
+
+    # Apply Crowdin API Key
+    with open(YAML_PATH, 'r') as crowdin_config_file:
+        crowdin_config = crowdin_config_file.read()
+    crowdin_config = re.sub(r'"api_token": "_secret_"',
+                            f'"api_token": "{API_KEY}"',
+                            crowdin_config, 1)
+    crowdin_config = re.sub(r'/_core_name_',
+                            f'/{CORE_NAME}'
+                            , crowdin_config)
+    with open(YAML_PATH, 'w') as crowdin_config_file:
+        crowdin_config_file.write(crowdin_config)
+
+    try:
+        # Download Crowdin CLI
+        jar_name = 'crowdin-cli.jar'
+        jar_path = t.os.path.join(DIR_PATH, jar_name)
+        crowdin_cli_file = 'crowdin-cli.zip'
+        crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file
+        crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file)
+
+        if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)):
+            print('download crowdin-cli.jar')
+            urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path)
+            with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref:
+                jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0])
+                for file in zip_ref.namelist():
+                    if file.endswith(jar_name):
+                        jar_file = file
+                        break
+                zip_ref.extract(jar_file, path=DIR_PATH)
+                os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path)
+                os.remove(crowdin_cli_path)
+                shutil.rmtree(jar_dir)
+
+        print('upload source *.json')
+        subprocess.run(['java', '-jar', jar_path, 'upload', 'sources', '--config', YAML_PATH])
+
+        # Reset Crowdin API Key
+        with open(YAML_PATH, 'r') as crowdin_config_file:
+            crowdin_config = crowdin_config_file.read()
+        crowdin_config = re.sub(r'"api_token": ".*?"',
+                                '"api_token": "_secret_"',
+                                crowdin_config, 1)
+
+        # TODO this is NOT safe!
+        crowdin_config = re.sub(re.escape(f'/{CORE_NAME}'),
+                                '/_core_name_',
+                                crowdin_config)
+
+        with open(YAML_PATH, 'w') as crowdin_config_file:
+            crowdin_config_file.write(crowdin_config)
+
+    except Exception as e:
+        # Try really hard to reset Crowdin API Key
+        with open(YAML_PATH, 'r') as crowdin_config_file:
+            crowdin_config = crowdin_config_file.read()
+        crowdin_config = re.sub(r'"api_token": ".*?"',
+                                '"api_token": "_secret_"',
+                                crowdin_config, 1)
+
+        # TODO this is NOT safe!
+        crowdin_config = re.sub(re.escape(f'/{CORE_NAME}'),
+                                '/_core_name_',
+                                crowdin_config)
+
+        with open(YAML_PATH, 'w') as crowdin_config_file:
+            crowdin_config_file.write(crowdin_config)
+        raise e

intl/crowdin_translate.py

+#!/usr/bin/env python3
+
+import core_option_translation as t
+
+
+if __name__ == '__main__':
+    try:
+        if t.os.path.isfile(t.sys.argv[1]):
+            _temp = t.os.path.dirname(t.sys.argv[1])
+        else:
+            _temp = t.sys.argv[1]
+        while _temp.endswith('/') or _temp.endswith('\\'):
+            _temp = _temp[:-1]
+        TARGET_DIR_PATH = _temp
+    except IndexError:
+        TARGET_DIR_PATH = t.os.path.dirname(t.os.path.dirname(t.os.path.realpath(__file__)))
+        print("No path provided, assuming parent directory:\n" + TARGET_DIR_PATH)
+
+    CORE_NAME = t.clean_file_name(t.sys.argv[2])
+    DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
+    LOCALISATIONS_PATH = t.os.path.join(DIR_PATH, CORE_NAME)
+    US_FILE_PATH = t.os.path.join(LOCALISATIONS_PATH, '_us.h')
+    H_FILE_PATH = t.os.path.join(TARGET_DIR_PATH, 'libretro_core_options.h')
+    INTL_FILE_PATH = t.os.path.join(TARGET_DIR_PATH, 'libretro_core_options_intl.h')
+
+    print('Getting texts from libretro_core_options.h')
+    with open(H_FILE_PATH, 'r+', encoding='utf-8') as _h_file:
+        _main_text = _h_file.read()
+    _hash_n_str = t.get_texts(_main_text)
+    _files = t.create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str)
+    _source_jsons = t.h2json(_files)
+
+    print('Converting translations *.json to *.h:')
+    localisation_files = t.os.scandir(LOCALISATIONS_PATH)
+    t.json2h(LOCALISATIONS_PATH, localisation_files)
+
+    print('Constructing libretro_core_options_intl.h')
+    t.create_intl_file(INTL_FILE_PATH, LOCALISATIONS_PATH, _main_text, _files["_us"])
+
+    print('\nAll done!')

intl/download_workflow.py

+#!/usr/bin/env python3
+
+import sys
+import subprocess
+
+try:
+    api_key = sys.argv[1]
+    core_name = sys.argv[2]
+    dir_path = sys.argv[3]
+except IndexError as e:
+    print('Please provide path to libretro_core_options.h, Crowdin API Token and core name!')
+    raise e
+
+subprocess.run(['python3', 'intl/core_option_translation.py', dir_path, core_name])
+subprocess.run(['python3', 'intl/initial_sync.py', api_key, core_name])
+subprocess.run(['python3', 'intl/crowdin_translate.py', dir_path, core_name])