From 1caca1dd9ff832122692b1df9b164dc2327319f2 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Fri, 8 Sep 2023 08:31:41 +0300 Subject: [PATCH] feat: add `atlas pull --expand-glob` to expand bash glob file patterns Example: 'atlas pull translations/*/done' pulls 'atlas pull translations/DoneXBlock/done' if it exists. --- README.rst | 5 ++ atlas | 56 +++++++++++++-- docs/decisions/0001-support-glob-pattern.rst | 71 ++++++++++++++++++++ example.atlas.yml | 1 + spec/config_spec.sh | 21 +++--- spec/integration_spec.sh | 65 ++++++++++++++++++ spec/pull_performance_spec.sh | 1 + spec/verbosity_spec.sh | 12 ++-- 8 files changed, 211 insertions(+), 21 deletions(-) create mode 100644 docs/decisions/0001-support-glob-pattern.rst diff --git a/README.rst b/README.rst index 5b6bd47..b9d6786 100644 --- a/README.rst +++ b/README.rst @@ -70,6 +70,7 @@ regularly and useful to understand ``atlas`` at a glance. branch: directory: : ... filter: ... + expand_glob: 0 Atlas can also use a configuration file in a different path using the `--config` flag after `atlas`: `atlas pull --config config.yml`. @@ -107,6 +108,10 @@ regularly and useful to understand ``atlas`` at a glance. `--filter=fr_CA,ar,es_419` will match both directories named 'es_419' and files named 'es_419.json' among others + `-g` or `--expand-glob`: + Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done' + if it exists. + Example: $ cd frontend-app-learning/src/i18n/messages diff --git a/atlas b/atlas index 30d90df..888391b 100755 --- a/atlas +++ b/atlas @@ -20,6 +20,7 @@ Configuration file: branch: directory: : ... filter: ... + expand_glob: 0 Atlas can also use a configuration file in a different path using the \`--config\` flag after \`atlas\`: \`atlas pull --config config.yml\`. @@ -57,6 +58,10 @@ Options: \`--filter=fr_CA,ar,es_419\` will match both directories named 'es_419' and files named 'es_419.json' among others + \`-g\` or \`--expand-glob\`: + Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done' + if it exists. + Example: $ cd frontend-app-learning/src/i18n/messages @@ -87,12 +92,13 @@ parser_definition_pull() { setup PULL_REST help:usage_pull -- \ "Usage: atlas pull [options...] [directory mappings...]" msg -- 'Options:' - param CONFIG --config -- "path to alternative atlas.yaml configuration file" - param BRANCH -b --branch -- "A branch of translation files" - param REPOSITORY -r --repository -- "The repository containing translation files" - param FILTER -f --filter -- "List of patterns to select which files and sub-directories to checkout." - flag VERBOSE -v --verbose -- "verbose output to terminal" - flag SILENT -s --silent -- "no output to terminal" + param CONFIG --config -- "path to alternative atlas.yaml configuration file" + param BRANCH -b --branch -- "A branch of translation files" + param REPOSITORY -r --repository -- "The repository containing translation files" + param FILTER -f --filter -- "List of patterns to select which files and sub-directories to checkout." + flag EXPAND_GLOB -g --expand-glob -- "Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done' if it exists." + flag VERBOSE -v --verbose -- "verbose output to terminal" + flag SILENT -s --silent -- "no output to terminal" disp :usage_pull -h --help } # @end @@ -165,6 +171,7 @@ Configuration file: branch: directory: : ... filter: ... + expand_glob: 0 Atlas can also use a configuration file in a different path using the `--config` flag after `atlas`: `atlas pull --config config.yml`. @@ -202,6 +209,10 @@ Options: `--filter=fr_CA,ar,es_419` will match both directories named 'es_419' and files named 'es_419.json' among others + `-g` or `--expand-glob`: + Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done' + if it exists. + Example: $ cd frontend-app-learning/src/i18n/messages @@ -238,6 +249,7 @@ CONFIG='' BRANCH='' REPOSITORY='' FILTER='' +EXPAND_GLOB='' VERBOSE='' SILENT='' PULL_REST='' @@ -252,7 +264,7 @@ parse_pull() { -[brf]?*) OPTARG=$1; shift eval 'set -- "${OPTARG%"${OPTARG#??}"}" "${OPTARG#??}"' ${1+'"$@"'} ;; - -[vsh]?*) OPTARG=$1; shift + -[gvsh]?*) OPTARG=$1; shift eval 'set -- "${OPTARG%"${OPTARG#??}"}" -"${OPTARG#??}"' ${1+'"$@"'} OPTARG= ;; esac @@ -277,6 +289,11 @@ parse_pull() { OPTARG=$2 FILTER="$OPTARG" shift ;; + '-g'|'--expand-glob') + [ "${OPTARG:-}" ] && OPTARG=${OPTARG#*\=} && set "noarg" "$1" && break + eval '[ ${OPTARG+x} ] &&:' && OPTARG='1' || OPTARG='' + EXPAND_GLOB="$OPTARG" + ;; '-v'|'--verbose') [ "${OPTARG:-}" ] && OPTARG=${OPTARG#*\=} && set "noarg" "$1" && break eval '[ ${OPTARG+x} ] &&:' && OPTARG='1' || OPTARG='' @@ -323,6 +340,7 @@ Options: -b, --branch BRANCH A branch of translation files -r, --repository REPOSITORY The repository containing translation files -f, --filter FILTER List of patterns to select which files and sub-directories to checkout. + -g, --expand-glob Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done' if it exists. -v, --verbose verbose output to terminal -s, --silent no output to terminal -h, --help @@ -368,6 +386,7 @@ set_pull_params() { pull_repository="openedx/openedx-translations" pull_branch="main" pull_filter="" + pull_expand_glob="" fi fi @@ -394,6 +413,12 @@ set_pull_params() { fi pull_filter="$(echo "$pull_filter" | tr ',' ' ')" # Accept comma and/or space separated directories list + + + if [ "$EXPAND_GLOB" ]; + then + pull_expand_glob="$EXPAND_GLOB" + fi } contains_substring() { @@ -416,6 +441,7 @@ display_pull_params() { echo " - repository: ${pull_repository:-Not Specified}" echo " - branch: ${pull_branch:-Not Specified}" echo " - filter: ${pull_filter:-Not Specified}" + echo " - expand-glob: ${pull_expand_glob:-Not Specified}" } check_git_version() { @@ -442,6 +468,16 @@ git_sparse_checkout_set() { xargs git sparse-checkout set } +expand_glob_pattern() { + # Expands glob pattern. + # An alternative to `compgen -G` that isn't available in non-Bash shells. + # Call: `expand_glob_pattern "translations/*/done"` to expand to `translations/DoneXBlock/done` if the file exists. + + # Shellcheck rule is disabled because we _want_ to expand the glob pattern. + # shellcheck disable=SC2086 + echo $1; +} + pull_translations() { check_git_version || return 1 @@ -540,6 +576,12 @@ pull_translations() { directory_from="$(echo "${directory_from_to}" | cut -f1 -d ':')" directory_from="./translations_TEMP/${directory_from}" + if [ -n "$pull_expand_glob" ]; + then + # Expand glob pattern e.g. `./translations_TEMP/*/done` to `./translations_TEMP/DoneXBlock/done` + directory_from="$(expand_glob_pattern "$directory_from")" + fi + # If no colon is provided then directory_to=. (aka working directory) if contains_substring "$directory_from_to" ":"; then diff --git a/docs/decisions/0001-support-glob-pattern.rst b/docs/decisions/0001-support-glob-pattern.rst new file mode 100644 index 0000000..326639b --- /dev/null +++ b/docs/decisions/0001-support-glob-pattern.rst @@ -0,0 +1,71 @@ +Establishing a Repository for Translations Management +##################################################### + +Description +*********** +Glob pattern support has been added to ``atlas``. + +Example +******* + +``atlas pull --expand-glob translations/*/done`` pulls +``atlas pull translations/DoneXBlock/done`` if it exists. + +This is useful for instance to pull XBlocks in which the directory name +isn't known but the module name is. + +This is an alternative to the `edx-platform-links`_ strategy because +``git sparse-checkout --no-cone`` don't support glob patterns across links. + + +Dismissed alternatives +********************** + +1. Support links again: +----------------------- + +Previous tests on unknown git version showed to support links in April 2023, +but it had no automated tests and broke somewhere during the +`git v2.25.1 version support`_ we've added. + +The sparse-checkout part is already complex, so we'll not touch it. + +2. Make complete copies of the translations in `edx-platform-links`_ +--------------------------------------------------------------------- + +The `edx-platform-links`_ directory is comprised of symbolic +links to the XBlock and plugins directories. This option would be to +replace the symbolic links with full copies of the XBlock and plugins +translations in a way that ``edx-platform-links/done`` is effectively +recreated with ``cp -r translations/DoneXBlock/done edx-platform-links/done`` +every time there's a update to the translations. + +Consequently, the `edx-platform-links`_ directory would be renamed to +``edx-platform-modules`` because it no longer contains links. + +This is the a viable and simple alternative to glob patterns, but _may_ +complicate the GitHub Transifex App sync process. + +3. Use the `edx-platform-links`_ to store the original files +------------------------------------------------------------ + +In this option, instead of storing symbolic links in the +`edx-platform-links`_ directory, we would store the original files and +setup Transifex to sync XBlocks and plugins to this directory. + +Consequently, the `edx-platform-links`_ directory would be renamed to +``edx-platform-modules`` because it no longer contains links. + +This would remove the symbolic links altogether resulting in +two translation root directories: + +- ``translations/``: contains the original files for all micro-frontends and + microservices by their GitHub repository name. +- ``translations/edx-platform-modules/``: contains the translations for + the edx-platform plugins and XBlocks by their Python module name. + +This is a viable option as well, but needs to refactor the Transifex +configuration which could reset the translations. + +.. _edx-platform-links: https://github.com/openedx/openedx-translations/blob/8a01424fd8f42e9e76aed34e235c82ab654cdfc5/translations/edx-platform-links/README.rst +.. _git v2.25.1 version support: https://github.com/openedx/openedx-atlas/pull/23 diff --git a/example.atlas.yml b/example.atlas.yml index bbe7250..b20335a 100644 --- a/example.atlas.yml +++ b/example.atlas.yml @@ -3,3 +3,4 @@ pull: directory: example_directory repository: example_repository filter: example_filter + expand_glob: 1 diff --git a/spec/config_spec.sh b/spec/config_spec.sh index def8a87..0441a05 100644 --- a/spec/config_spec.sh +++ b/spec/config_spec.sh @@ -28,12 +28,13 @@ Describe 'Test Default Config' It 'sets the default values correctly when no config param is passed and atlas.yml does not exist' When run source ./atlas pull - The lines of output should equal 5 + The lines of output should equal 6 The line 1 of output should equal 'Pulling translation files' The line 2 of output should equal ' - directory: Not Specified' The line 3 of output should equal ' - repository: openedx/openedx-translations' The line 4 of output should equal ' - branch: main' The line 5 of output should equal ' - filter: Not Specified' + The line 6 of output should equal ' - expand-glob: Not Specified' The variable PULL_TRANSLATIONS_CALLED should equal true End End @@ -55,12 +56,13 @@ Describe 'Test example atlas.yml' It 'reads atlas.yml correctly when no config param is passed' When run source ./atlas pull - The lines of output should equal 5 + The lines of output should equal 6 The line 1 of output should equal 'Pulling translation files' The line 2 of output should equal ' - directory: example_directory' The line 3 of output should equal ' - repository: example_repository' The line 4 of output should equal ' - branch: example_branch' The line 5 of output should equal ' - filter: example_filter' + The line 6 of output should equal ' - expand-glob: 1' The variable PULL_TRANSLATIONS_CALLED should equal true End End @@ -77,12 +79,13 @@ Describe 'Test example.atlas.yml' It 'reads example.atlas.yml correctly when passed as config param' When run source ./atlas pull --config example.atlas.yml - The lines of output should equal 5 + The lines of output should equal 6 The line 1 of output should equal 'Pulling translation files' The line 2 of output should equal ' - directory: example_directory' The line 3 of output should equal ' - repository: example_repository' The line 4 of output should equal ' - branch: example_branch' The line 5 of output should equal ' - filter: example_filter' + The line 6 of output should equal ' - expand-glob: 1' The variable PULL_TRANSLATIONS_CALLED should equal true End End @@ -98,13 +101,14 @@ Describe 'Test full flags' } It 'correctly reads full flag params' - When run source ./atlas pull --repository full_flag_repository --branch full_flag_branch --filter ar,es_419 positional_arg_directory:to_dir - The lines of output should equal 5 + When run source ./atlas pull --repository full_flag_repository --branch full_flag_branch --expand-glob --filter ar,es_419 positional_arg_directory:to_dir + The lines of output should equal 6 The line 1 of output should equal 'Pulling translation files' The line 2 of output should equal ' - directory: positional_arg_directory:to_dir' The line 3 of output should equal ' - repository: full_flag_repository' The line 4 of output should equal ' - branch: full_flag_branch' The line 5 of output should equal ' - filter: ar es_419' + The line 6 of output should equal ' - expand-glob: 1' The variable PULL_TRANSLATIONS_CALLED should equal true End End @@ -120,13 +124,14 @@ Describe 'Test short flags' } It 'correctly reads short flag params' - When run source ./atlas pull -r short_flag_repository -b short_flag_branch -f 'ar es_419' positional_arg_directory:mapped_to_dir - The lines of output should equal 5 + When run source ./atlas pull -r short_flag_repository -b short_flag_branch -g -f 'ar es_419' positional_arg_directory:mapped_to_dir + The lines of output should equal 6 The line 1 of output should equal 'Pulling translation files' The line 2 of output should equal ' - directory: positional_arg_directory:mapped_to_dir' The line 3 of output should equal ' - repository: short_flag_repository' The line 4 of output should equal ' - branch: short_flag_branch' The line 5 of output should equal ' - filter: ar es_419' + The line 6 of output should equal ' - expand-glob: 1' The variable PULL_TRANSLATIONS_CALLED should equal true End End @@ -146,7 +151,7 @@ Describe 'Test short flags' When run source ./atlas pull -r short_flag_repository \ orange_dir:foo_local_dir \ blue_dir:bazz_local_dir - The lines of output should equal 5 + The lines of output should equal 6 The line 1 of output should equal 'Pulling translation files' The line 2 of output should equal ' - directory: orange_dir:foo_local_dir blue_dir:bazz_local_dir' The variable PULL_TRANSLATIONS_CALLED should equal true diff --git a/spec/integration_spec.sh b/spec/integration_spec.sh index 98d966c..cd1a9d8 100644 --- a/spec/integration_spec.sh +++ b/spec/integration_spec.sh @@ -1,3 +1,5 @@ +# Integration tests to run across multiple git versions. + check_call_time() { # Custom ShellSpec assert for call time. local test_name="$1" @@ -46,6 +48,7 @@ Describe 'Pull performance on edX Platform Arabic translations' - repository: openedx/xblock-drag-and-drop-v2 - branch: v3.2.0 - filter: ar fr + - expand-glob: Not Specified Creating a temporary Git repository to pull translations into "./translations_TEMP"... Done. Setting git sparse-checkout rules... @@ -70,3 +73,65 @@ translations_TEMP/drag_and_drop_v2/conf/locale/fr/LC_MESSAGES/text.mo translations_TEMP/drag_and_drop_v2/conf/locale/fr/LC_MESSAGES/text.po" End End + + + +Describe 'Bash glob patterns' + Intercept begin_pull_translations_mock + __begin_pull_translations_mock__() { + cp() { + echo /usr/bin/env cp $@ # Run cp, but intercept with `find` + + # The output is sorted to ensure cross-platform consistent output + find translations_TEMP/drag_and_drop_v2/conf -type f | LC_ALL=C sort -n -k1,1 >&2 # Allow checking tree content. + } + } + + setup() { + TEST_START_TIME=$(date +%s) # Time the bash script + PREVIOUS_PWD="$PWD"; + cd "$(mktemp -d)" || exit 1 # Run in a temp. directory + } + tearDown() { + cd "$PREVIOUS_PWD" || exit 1`` + } + BeforeEach 'setup' + AfterEach 'tearDown' + + It 'pulls with glob patterns' + TEST_START_TIME=$(date +%s) # Time the bash script + When run source $PREVIOUS_PWD/atlas pull -r openedx/xblock-drag-and-drop-v2 -b v3.2.0 --expand-glob -f ar 'drag_and_drop_v2/*/locale:drag_and_drop_v2' + Assert check_call_time "Pull xblock-drag-and-drop-v2" $TEST_START_TIME 60 # Allow a maximum of 60 seconds + + The output should equal 'Pulling translation files + - directory: drag_and_drop_v2/*/locale:drag_and_drop_v2 + - repository: openedx/xblock-drag-and-drop-v2 + - branch: v3.2.0 + - filter: ar + - expand-glob: 1 +Creating a temporary Git repository to pull translations into "./translations_TEMP"... +Done. +Setting git sparse-checkout rules... +Done. +Pulling translation files from the repository... +Done. +Copying translations from "./translations_TEMP/drag_and_drop_v2/conf/locale" to "./drag_and_drop_v2"... +/usr/bin/env cp -r ./translations_TEMP/drag_and_drop_v2/conf/locale/ar drag_and_drop_v2/ +Done. +Removing temporary directory... +Done. + +Translations pulled successfully!' + + # Checks the results of `find` + The error should equal "translations_TEMP/drag_and_drop_v2/conf/locale/ar/LC_MESSAGES/text.mo +translations_TEMP/drag_and_drop_v2/conf/locale/ar/LC_MESSAGES/text.po" + End + + + It 'will not expand glob pattern unless specified' + When run source $PREVIOUS_PWD/atlas pull -r openedx/xblock-drag-and-drop-v2 -b v3.2.0 -f ar 'drag_and_drop_v2/*/locale:drag_and_drop_v2' + + The output should include 'Skipped copying "./translations_TEMP/drag_and_drop_v2/*/locale" because it was not found in the repository.' + End +End diff --git a/spec/pull_performance_spec.sh b/spec/pull_performance_spec.sh index 1538f4d..d9b4907 100644 --- a/spec/pull_performance_spec.sh +++ b/spec/pull_performance_spec.sh @@ -46,6 +46,7 @@ Describe 'Pull performance on edX Platform Arabic translations' - repository: openedx/edx-platform - branch: open-release/nutmeg.1 - filter: ar fr + - expand-glob: Not Specified Creating a temporary Git repository to pull translations into "./translations_TEMP"... Done. Setting git sparse-checkout rules... diff --git a/spec/verbosity_spec.sh b/spec/verbosity_spec.sh index 3d1b166..458eb13 100644 --- a/spec/verbosity_spec.sh +++ b/spec/verbosity_spec.sh @@ -60,8 +60,8 @@ Describe 'Test default verbosity' CD_CALLED=false RM_CALLED=false When run source ./atlas pull dir_from:dir_to - The lines of output should equal 17 - The line 17 of output should equal 'Translations pulled successfully!' + The lines of output should equal 18 + The line 18 of output should equal 'Translations pulled successfully!' The variable GIT_CALLED_QUIETLY should equal true The variable GIT_CALLED_NOT_QUIETLY should equal false The variable GIT_CALLED_VERBOSELY should equal false @@ -214,8 +214,8 @@ Describe 'Test verbose flag' CD_CALLED=false RM_CALLED=false When run source ./atlas pull --verbose dir_from:dir_to - The lines of output should equal 17 - The line 17 of output should equal 'Translations pulled successfully!' + The lines of output should equal 18 + The line 18 of output should equal 'Translations pulled successfully!' The variable GIT_CALLED_QUIETLY should equal false The variable GIT_CALLED_NOT_QUIETLY should equal true The variable GIT_CALLED_VERBOSELY should equal true @@ -234,8 +234,8 @@ Describe 'Test verbose flag' CD_CALLED=false RM_CALLED=false When run source ./atlas pull -v dir_from:dir_to - The lines of output should equal 17 - The line 17 of output should equal 'Translations pulled successfully!' + The lines of output should equal 18 + The line 18 of output should equal 'Translations pulled successfully!' The variable GIT_CALLED_QUIETLY should equal false The variable GIT_CALLED_NOT_QUIETLY should equal true The variable GIT_CALLED_VERBOSELY should equal true