feat: add atlas pull --expand-glob to expand bash glob file patterns

Example: 'atlas pull translations/*/done' pulls 'atlas pull translations/DoneXBlock/done' if it exists.
openedx · Sep 8, 2023 · cdd49a6 · cdd49a6
1 parent dd55d89
commit cdd49a6
Show file tree

Hide file tree

Showing 8 changed files with 194 additions and 21 deletions.
diff --git a/README.rst b/README.rst
@@ -70,6 +70,7 @@ regularly and useful to understand ``atlas`` at a glance.
           branch: <branch-name>
           directory: <repo-directory-name>:<local-dir-name> ...
           filter: <pattern> ...
+          expand_glob: 0
 
         Atlas can also use a configuration file in a different path using the `--config` flag
         after `atlas`: `atlas pull --config config.yml`.
@@ -107,6 +108,10 @@ regularly and useful to understand ``atlas`` at a glance.
            `--filter=fr_CA,ar,es_419` will match both directories named 'es_419' and
            files named 'es_419.json' among others
 
+       `-g` or `--expand-glob`:
+           Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done'
+           if it exists.
+
     Example:
 
         $ cd frontend-app-learning/src/i18n/messages

diff --git a/atlas b/atlas
@@ -20,6 +20,7 @@ Configuration file:
       branch: <branch-name>
       directory: <repo-directory-name>:<local-dir-name> ...
       filter: <pattern> ...
+      expand_glob: 0
 
     Atlas can also use a configuration file in a different path using the \`--config\` flag
     after \`atlas\`: \`atlas pull --config config.yml\`.
@@ -57,6 +58,10 @@ Options:
        \`--filter=fr_CA,ar,es_419\` will match both directories named 'es_419' and
        files named 'es_419.json' among others
 
+   \`-g\` or \`--expand-glob\`:
+       Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done'
+       if it exists.
+
 Example:
 
     $ cd frontend-app-learning/src/i18n/messages
@@ -87,12 +92,13 @@ parser_definition_pull() {
 	setup   PULL_REST help:usage_pull -- \
 		"Usage: atlas pull [options...] [directory mappings...]"
 	msg -- 'Options:'
-	param		CONFIG			--config			-- "path to alternative atlas.yaml configuration file"
-	param		BRANCH		-b	--branch			-- "A branch of translation files"
-	param		REPOSITORY	-r	--repository		-- "The repository containing translation files"
-	param		FILTER	    -f	--filter			-- "List of patterns to select which files and sub-directories to checkout."
-	flag		VERBOSE		-v	--verbose			-- "verbose output to terminal"
-	flag		SILENT		-s	--silent			-- "no output to terminal"
+	param		CONFIG			--config			    -- "path to alternative atlas.yaml configuration file"
+	param		BRANCH		    -b	--branch			-- "A branch of translation files"
+	param		REPOSITORY	    -r	--repository		-- "The repository containing translation files"
+	param		FILTER	        -f	--filter			-- "List of patterns to select which files and sub-directories to checkout."
+	flag		EXPAND_GLOB     -g	--expand-glob       -- "Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done' if it exists."
+	flag		VERBOSE		    -v	--verbose			-- "verbose output to terminal"
+	flag		SILENT		    -s	--silent			-- "no output to terminal"
 	disp    :usage_pull  -h --help
 }
 # @end
@@ -165,6 +171,7 @@ Configuration file:
       branch: <branch-name>
       directory: <repo-directory-name>:<local-dir-name> ...
       filter: <pattern> ...
+      expand_glob: 0
 
     Atlas can also use a configuration file in a different path using the `--config` flag
     after `atlas`: `atlas pull --config config.yml`.
@@ -202,6 +209,10 @@ Options:
        `--filter=fr_CA,ar,es_419` will match both directories named 'es_419' and
        files named 'es_419.json' among others
 
+   `-g` or `--expand-glob`:
+       Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done'
+       if it exists.
+
 Example:
 
     $ cd frontend-app-learning/src/i18n/messages
@@ -238,6 +249,7 @@ CONFIG=''
 BRANCH=''
 REPOSITORY=''
 FILTER=''
+EXPAND_GLOB=''
 VERBOSE=''
 SILENT=''
 PULL_REST=''
@@ -252,7 +264,7 @@ parse_pull() {
       -[brf]?*) OPTARG=$1; shift
         eval 'set -- "${OPTARG%"${OPTARG#??}"}" "${OPTARG#??}"' ${1+'"$@"'}
         ;;
-      -[vsh]?*) OPTARG=$1; shift
+      -[gvsh]?*) OPTARG=$1; shift
         eval 'set -- "${OPTARG%"${OPTARG#??}"}" -"${OPTARG#??}"' ${1+'"$@"'}
         OPTARG= ;;
     esac
@@ -277,6 +289,11 @@ parse_pull() {
         OPTARG=$2
         FILTER="$OPTARG"
         shift ;;
+      '-g'|'--expand-glob')
+        [ "${OPTARG:-}" ] && OPTARG=${OPTARG#*\=} && set "noarg" "$1" && break
+        eval '[ ${OPTARG+x} ] &&:' && OPTARG='1' || OPTARG=''
+        EXPAND_GLOB="$OPTARG"
+        ;;
       '-v'|'--verbose')
         [ "${OPTARG:-}" ] && OPTARG=${OPTARG#*\=} && set "noarg" "$1" && break
         eval '[ ${OPTARG+x} ] &&:' && OPTARG='1' || OPTARG=''
@@ -323,6 +340,7 @@ Options:
   -b, --branch BRANCH         A branch of translation files
   -r, --repository REPOSITORY The repository containing translation files
   -f, --filter FILTER         List of patterns to select which files and sub-directories to checkout.
+  -g, --expand-glob           Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done' if it exists.
   -v, --verbose               verbose output to terminal
   -s, --silent                no output to terminal
   -h, --help                  
@@ -368,6 +386,7 @@ set_pull_params() {
       pull_repository="openedx/openedx-translations"
       pull_branch="main"
       pull_filter=""
+      pull_expand_glob=""
     fi
   fi
 
@@ -394,6 +413,12 @@ set_pull_params() {
   fi
 
   pull_filter="$(echo "$pull_filter" | tr ',' ' ')"  # Accept comma and/or space separated directories list
+
+
+  if [ "$EXPAND_GLOB" ];
+  then
+      pull_expand_glob="$EXPAND_GLOB"
+  fi
 }
 
 contains_substring() {
@@ -416,6 +441,7 @@ display_pull_params() {
   echo " - repository: ${pull_repository:-Not Specified}"
   echo " - branch: ${pull_branch:-Not Specified}"
   echo " - filter: ${pull_filter:-Not Specified}"
+  echo " - expand-glob: ${pull_expand_glob:-Not Specified}"
 }
 
 check_git_version() {
@@ -442,6 +468,16 @@ git_sparse_checkout_set() {
     xargs git sparse-checkout set
 }
 
+expand_glob_pattern() {
+    # Expands glob pattern.
+    # An alternative to `compgen -G` that isn't available in non-Bash shells.
+    # Call: `expand_glob_pattern "translations/*/done"` to expand to `translations/DoneXBlock/done` if the file exists.
+
+    # Shellcheck rule is disabled because we _want_ to expand the glob pattern.
+    # shellcheck disable=SC2086
+    echo $1;
+}
+
 pull_translations() {
   check_git_version || return 1
 
@@ -540,6 +576,12 @@ pull_translations() {
     directory_from="$(echo "${directory_from_to}" | cut -f1 -d ':')"
     directory_from="./translations_TEMP/${directory_from}"
 
+    if [ -n "$pull_expand_glob" ];
+    then
+      # Expand glob pattern e.g. `./translations_TEMP/*/done` to `./translations_TEMP/DoneXBlock/done`
+      directory_from="$(expand_glob_pattern "$directory_from")"
+    fi
+
     # If no colon is provided then directory_to=. (aka working directory)
     if contains_substring "$directory_from_to" ":";
     then

diff --git a/docs/decisions/0001-support-glob-pattern.rst b/docs/decisions/0001-support-glob-pattern.rst
@@ -0,0 +1,54 @@
+Establishing a Repository for Translations Management
+#####################################################
+
+Description
+***********
+Glob pattern support has been added to ``atlas``.
+
+Example
+*******
+
+``atlas pull --expand-glob translations/*/done`` pulls
+``atlas pull translations/DoneXBlock/done`` if it exists.
+
+This is useful for instance to pull XBlocks in which the directory name
+isn't known but the module name is.
+
+This is an alternative to the `edx-platform-links`_ strategy because
+``git sparse-checkout --no-cone`` don't support glob patterns across links.
+
+
+Dismissed alternatives
+**********************
+
+1. Support links again:
+-----------------------
+
+Previous tests on unknown git version showed to support links in April 2023,
+but it had no automated tests and broke somewhere during the
+`git v2.25.1 version support`_ we've added.
+
+The sparse-checkout part is already complex, so we'll not touch it.
+
+2. Make complete copies of the translations in `edx-platform-links`_
+---------------------------------------------------------------------
+This is the most viable and simple alternative to glob patterns.
+
+If it turns out the ``--expand-glob`` option is flaky to work with, we'll
+fallback to this.
+
+This _may_ complicate the GitHub Transifex App sync process which we prefer
+to keep simple and avoid adding additional step to ensure the
+`edx-platform-links`_ have full copies of the XBlock and plugins translations
+instead of symbolic links.
+
+3. Use the `edx-platform-links`_ to store the original files
+------------------------------------------------------------
+
+This is a viable option as well but needs to refactor the Transifex
+configuration which could reset the translations.
+
+Therefore, we'll only fallback to this if the glob patterns has problems.
+
+.. _edx-platform-links: https://github.com/openedx/openedx-translations/blob/8a01424fd8f42e9e76aed34e235c82ab654cdfc5/translations/edx-platform-links/README.rst
+.. _git v2.25.1 version support: https://github.com/openedx/openedx-atlas/pull/23
diff --git a/example.atlas.yml b/example.atlas.yml
@@ -3,3 +3,4 @@ pull:
   directory: example_directory
   repository: example_repository
   filter: example_filter
+  expand_glob: 1
diff --git a/spec/config_spec.sh b/spec/config_spec.sh
@@ -28,12 +28,13 @@ Describe 'Test Default Config'
 
   It 'sets the default values correctly when no config param is passed and atlas.yml does not exist'
     When run source ./atlas pull
-    The lines of output should equal 5
+    The lines of output should equal 6
     The line 1 of output should equal 'Pulling translation files'
     The line 2 of output should equal ' - directory: Not Specified'
     The line 3 of output should equal ' - repository: openedx/openedx-translations'
     The line 4 of output should equal ' - branch: main'
     The line 5 of output should equal ' - filter: Not Specified'
+    The line 6 of output should equal ' - expand-glob: Not Specified'
     The variable PULL_TRANSLATIONS_CALLED should equal true
   End
 End
@@ -55,12 +56,13 @@ Describe 'Test example atlas.yml'
 
   It 'reads atlas.yml correctly when no config param is passed'
     When run source ./atlas pull
-    The lines of output should equal 5
+    The lines of output should equal 6
     The line 1 of output should equal 'Pulling translation files'
     The line 2 of output should equal ' - directory: example_directory'
     The line 3 of output should equal ' - repository: example_repository'
     The line 4 of output should equal ' - branch: example_branch'
     The line 5 of output should equal ' - filter: example_filter'
+    The line 6 of output should equal ' - expand-glob: 1'
     The variable PULL_TRANSLATIONS_CALLED should equal true
   End
 End
@@ -77,12 +79,13 @@ Describe 'Test example.atlas.yml'
 
   It 'reads example.atlas.yml correctly when passed as config param'
     When run source ./atlas pull --config example.atlas.yml
-    The lines of output should equal 5
+    The lines of output should equal 6
     The line 1 of output should equal 'Pulling translation files'
     The line 2 of output should equal ' - directory: example_directory'
     The line 3 of output should equal ' - repository: example_repository'
     The line 4 of output should equal ' - branch: example_branch'
     The line 5 of output should equal ' - filter: example_filter'
+    The line 6 of output should equal ' - expand-glob: 1'
     The variable PULL_TRANSLATIONS_CALLED should equal true
   End
 End
@@ -98,13 +101,14 @@ Describe 'Test full flags'
   }
 
   It 'correctly reads full flag params'
-    When run source ./atlas pull --repository full_flag_repository --branch full_flag_branch --filter ar,es_419 positional_arg_directory:to_dir
-    The lines of output should equal 5
+    When run source ./atlas pull --repository full_flag_repository --branch full_flag_branch --expand-glob --filter ar,es_419 positional_arg_directory:to_dir
+    The lines of output should equal 6
     The line 1 of output should equal 'Pulling translation files'
     The line 2 of output should equal ' - directory: positional_arg_directory:to_dir'
     The line 3 of output should equal ' - repository: full_flag_repository'
     The line 4 of output should equal ' - branch: full_flag_branch'
     The line 5 of output should equal ' - filter: ar es_419'
+    The line 6 of output should equal ' - expand-glob: 1'
     The variable PULL_TRANSLATIONS_CALLED should equal true
   End
 End
@@ -120,13 +124,14 @@ Describe 'Test short flags'
   }
 
   It 'correctly reads short flag params'
-    When run source ./atlas pull -r short_flag_repository -b short_flag_branch -f 'ar es_419' positional_arg_directory:mapped_to_dir
-    The lines of output should equal 5
+    When run source ./atlas pull -r short_flag_repository -b short_flag_branch -g -f 'ar es_419' positional_arg_directory:mapped_to_dir
+    The lines of output should equal 6
     The line 1 of output should equal 'Pulling translation files'
     The line 2 of output should equal ' - directory: positional_arg_directory:mapped_to_dir'
     The line 3 of output should equal ' - repository: short_flag_repository'
     The line 4 of output should equal ' - branch: short_flag_branch'
     The line 5 of output should equal ' - filter: ar es_419'
+    The line 6 of output should equal ' - expand-glob: 1'
     The variable PULL_TRANSLATIONS_CALLED should equal true
   End
 End
@@ -146,7 +151,7 @@ Describe 'Test short flags'
     When run source ./atlas pull -r short_flag_repository  \
                             orange_dir:foo_local_dir \
                             blue_dir:bazz_local_dir
-    The lines of output should equal 5
+    The lines of output should equal 6
     The line 1 of output should equal 'Pulling translation files'
     The line 2 of output should equal ' - directory: orange_dir:foo_local_dir blue_dir:bazz_local_dir'
     The variable PULL_TRANSLATIONS_CALLED should equal true