WV 2612 verbose mode 2 (#4521)

* buildOptions * add verbose scripts * getCapabilities * getVisMetadata * validateConfigs * revert getVisMetadata * move ident into conditional block
nasa-gibs · Jul 28, 2023 · fd7ed28 · fd7ed28
1 parent 825edc2
commit fd7ed28
Show file tree

Hide file tree

Showing 4 changed files with 92 additions and 14 deletions.
diff --git a/package.json b/package.json
@@ -28,8 +28,9 @@
   "scripts": {
     "analyze": "cross-env ANALYZE_MODE=true NODE_ENV=production webpack",
     "build": "run-s clean getcapabilities build:config build:dev",
+    "build:verbose": "run-s clean getcapabilities:verbose build:config build:dev",
     "build:ci": "run-s getcapabilities build:config build:prod",
-    "build:options": "bash -c 'tasks/buildOptions.sh'",
+    "build:options": "bash tasks/buildOptions.sh",
     "build:config": "run-s build:options && node ./tasks/util/config.js",
     "build:dev": "cross-env NODE_ENV=development webpack",
     "build:prod": "cross-env NODE_ENV=production webpack",
@@ -47,6 +48,7 @@
     "docker:stop": "docker stop -t 1 worldview ; true",
     "e2e": "npm run docker:restart && docker exec -it worldview /bin/bash /build/e2e/docker-ci.sh ; npm run docker:stop",
     "getcapabilities": "bash -c 'FETCH_GC=1 npm run build:options'",
+    "getcapabilities:verbose": "bash -c 'FETCH_GC=1 npm run build:options -- -v'",
     "generatePreviews": "node ./tasks/build-options/fetchPreviewSnapshots.py",
     "linkcheck": "node ./tasks/link-check",
     "lint": "run-s lint:js lint:scss",

diff --git a/tasks/build-options/getCapabilities.js b/tasks/build-options/getCapabilities.js
@@ -24,6 +24,12 @@ const options = yargs
     type: 'string',
     description: 'getcapabilities file'
   })
+  .option('mode', {
+    demandOption: true,
+    alias: 'm',
+    type: 'string',
+    description: 'mode'
+  })
   .epilog('Pulls GetCapabilities XML and linked metadata from configured locations')
 
 const { argv } = options
@@ -68,7 +74,9 @@ async function getCapabilities () {
       const inputFile = value.from
       const outputFile = `${outputDir}/${value.to}`
 
+      if (argv.mode === 'verbose') console.warn(`Fetching config for ${inputFile} to ${outputFile}...`)
       await fetchConfigs(inputFile, outputFile)
+      if (argv.mode === 'verbose') console.warn(`Processing capabilities for ${outputFile}...`)
       await processGetCapabilities(outputFile)
     }
   }
@@ -83,6 +91,7 @@ async function fetchConfigs (inputFile, outputFile) {
     responseType: 'stream',
     timeout: 10000
   }).then(async (response) => {
+    if (argv.mode === 'verbose') console.warn(`Writing ${outputFile}...`)
     await response.data.pipe(writer)
     return finished(writer)
   })
@@ -99,10 +108,15 @@ async function handleException (error, link, dir, ext, count) {
 }
 
 async function processVectorData (layer) {
+  if (argv.mode === 'verbose') {
+    const ident = layer['ows:Identifier']._text
+    console.warn(`Processing vector data for ${ident}:`)
+  }
   if (layer['ows:Metadata']) {
     Object.values(layer['ows:Metadata']).forEach((item) => {
       const schemaVersion = item._attributes['xlink:role']
       if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/layer/1.0') {
+        if (argv.mode === 'verbose') console.warn(`  Processing Metadata: ${item._attributes['xlink:href']}`)
         const vectorDataLink = item._attributes['xlink:href']
         const vectorDataFile = path.basename(vectorDataLink)
         const vectorDataId = path.parse(vectorDataFile).name
@@ -114,11 +128,13 @@ async function processVectorData (layer) {
 
 async function processLayer (layer) {
   const ident = layer['ows:Identifier']._text
+  if (argv.mode === 'verbose') console.warn(`Processing layer ${ident}:`)
   if (layer['ows:Metadata']) {
     if (config.skipPalettes) {
       console.warn(`${prog}: WARN: Skipping palette for ${ident} \n`)
     } else {
       Object.values(layer['ows:Metadata']).forEach((item) => {
+        if (argv.mode === 'verbose') console.warn(`  Processing pallette: ${item._attributes['xlink:href']}`)
         const schemaVersion = item._attributes['xlink:role']
         if (schemaVersion === 'http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.3') {
           const colormapLink = item._attributes['xlink:href']

diff --git a/tasks/build-options/validateConfigs.js b/tasks/build-options/validateConfigs.js
@@ -22,6 +22,12 @@ const options = yargs
     type: 'string',
     description: 'layer-config.json schema'
   })
+  .option('mode', {
+    demandOption: true,
+    alias: 'm',
+    type: 'string',
+    description: 'mode'
+  })
   .epilog('Validates layers using a JSON schema')
 
 const { argv } = options
@@ -47,13 +53,15 @@ async function main () {
     validateFile(filePath)
   }
   if (invalidJsonFiles.length) {
+    if (argv.mode === 'verbose') console.warn(`${prog}: Invalid JSON files: ${invalidJsonFiles}`)
     throw new Error(`${prog}: FAILED: ${invalidJsonFiles.length} layer configs failed validation.`)
   } else {
     console.warn(`${prog}: PASSED: All layer configs passed validation!`)
   }
 }
 
 async function validateFile (filePath) {
+  if (argv.mode === 'verbose') console.warn(`${prog}: Validating ${filePath}`)
   const layerFile = fs.readFileSync(filePath)
   const layer = JSON.parse(layerFile)
   const valid = validate(layer)

diff --git a/tasks/buildOptions.sh b/tasks/buildOptions.sh
@@ -1,14 +1,22 @@
 #!/bin/bash
-
 PROG=$(basename "$0")
 BASE=$(dirname "$0")/..
-
 SRC_DIR="$BASE/config/default"
 OPT_DIR="$BASE/config/default"
 BUILD_DIR="$BASE/build/options-build"
 DEST_DIR="$BASE/build/options"
 SCRIPTS_DIR="$BASE/tasks/build-options"
 
+MODE="default"
+
+while getopts ":v" option; do
+   echo "Option -$option set"
+   case $option in
+      v)
+       echo "Verbose Mode Activated"
+       MODE="verbose";;
+   esac
+done
 
 # If there is an active directory, use instead of defaults
 if [ -d "$BASE/config/active" ]; then
@@ -50,71 +58,100 @@ mkdir -p "$BUILD_DIR/colormaps"
 # If $FETCH_GC is set, make various API requests
 if [ "$FETCH_GC" ] ; then
     # Fetch GC files and create colormaps, vectordata and vectorstyle files
+    if (( $MODE = "verbose" )) ; then
+      echo "Fetch GC files and create colormaps, vectordata and vectorstyle files"
+    fi
     rm -rf "$OPT_DIR/$OPT_SUBDIR/gc/*"
     rm -rf "$OPT_DIR/$OPT_SUBDIR/colormaps/gc/*"
     `node $SCRIPTS_DIR/getCapabilities.js \
       --config "$OPT_DIR/$OPT_SUBDIR/config.json" \
-      --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc"`
+      --getcapabilities "$OPT_DIR/$OPT_SUBDIR/gc" \
+      --mode "$MODE"`
 
     # Get metadata for files in layerOrder.json and combine this data into 1 file
+    if (( $MODE = "verbose" )) ; then
+      echo "Get metadata for files in layerOrder.json and combine this data into 1 file"
+    fi
     rm -rf "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
     mkdir -p "$OPT_DIR/$OPT_SUBDIR/layer-metadata"
     `node $SCRIPTS_DIR/getVisMetadata.js \
       --features "$BUILD_DIR/features.json" \
       --layerOrder "$BUILD_DIR/config/wv.json/layerOrder.json" \
-      --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json"`
+      --layerMetadata "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" \
+      --mode "$MODE"`
 else
   # Validate layers in wv.json with a JSON schema
+  if (( $MODE = "verbose" )) ; then
+    echo "Validate layers in wv.json with a JSON schema"
+  fi
   `node $SCRIPTS_DIR/validateConfigs.js \
     --inputDirectory "$SRC_DIR/common/config/wv.json/layers" \
-    --schemaFile "$BASE/schemas/layer-config.json"`
+    --schemaFile "$BASE/schemas/layer-config.json" \
+    --mode "$MODE"`
 
   if [ -e "$BUILD_DIR/features.json" ] ; then
       cp "$BUILD_DIR/features.json" "$BUILD_DIR/config/wv.json/_features.json"
   fi
 
   # Run extractConfigFromWMTS.js script with config.json
   if [ -e "$BUILD_DIR/config.json" ] ; then
+    if (( $MODE = "verbose" )) ; then
+      echo "Run extractConfigFromWMTS.js script with config.json"
+    fi
     `node $SCRIPTS_DIR/extractConfigFromWMTS.js \
       --config "$BUILD_DIR/config.json" \
       --inputDir "$BUILD_DIR/gc" \
-      --outputDir  "$BUILD_DIR/_wmts"`
+      --outputDir  "$BUILD_DIR/_wmts" \
+      --mode "$MODE"`
   fi
 
   # Run processVectorStyles.js and move vectorstyles where we want them
   if [ -e "$BUILD_DIR/gc/vectorstyles" ] ; then
+      if (( $MODE = "verbose" )) ; then
+        echo "Run processVectorStyles.js and move vectorstyles where we want them"
+      fi
       mkdir -p "$BUILD_DIR/config/wv.json/vectorstyles"
       `node $SCRIPTS_DIR/processVectorStyles.js \
         --inputDir "$BUILD_DIR/gc/vectorstyles" \
-        --outputDir "$BUILD_DIR/config/wv.json/vectorstyles"`
+        --outputDir "$BUILD_DIR/config/wv.json/vectorstyles" \
+        --mode "$MODE"`
   fi
 
   # Run processVectorData.js and move vectordata where we want them
   if [ -e "$BUILD_DIR/gc/vectordata" ] ; then
+      if (( $MODE = "verbose" )) ; then
+        echo "Run processVectorData.js and move vectordata where we want them"
+      fi
       mkdir -p "$BUILD_DIR/config/wv.json/vectordata"
       `node $SCRIPTS_DIR/processVectorData.js \
         --inputDir "$BUILD_DIR/gc/vectordata" \
-        --outputDir "$BUILD_DIR/config/wv.json/vectordata"`
+        --outputDir "$BUILD_DIR/config/wv.json/vectordata" \
+        --mode "$MODE"`
   fi
 
   # Run processColormap.js and move colormaps where we want them
   if [ -e "$BUILD_DIR/colormaps" ] ; then
+      if (( $MODE = "verbose" )) ; then
+        echo "Run processColormap.js and move colormaps where we want them"
+      fi
       mkdir -p "$BUILD_DIR"/config/palettes
       if [ -d "$BUILD_DIR"/gc/colormaps ] ; then
           cp -r "$BUILD_DIR"/gc/colormaps "$BUILD_DIR"/colormaps/gc
       fi
       `node $SCRIPTS_DIR/processColormap.js \
         --config "$OPT_DIR/$OPT_SUBDIR/config.json" \
         --inputDir "$BUILD_DIR/colormaps" \
-        --outputDir "$BUILD_DIR/config/palettes"`
+        --outputDir "$BUILD_DIR/config/palettes" \
+        --mode "$MODE"`
   fi
 
   # Throw error if no categoryGroupOrder.json file present
   if [ ! -e "$BUILD_DIR/config/wv.json/categoryGroupOrder.json" ] ; then
       echo "categoryGroupOrder.json not found.  Generating..."
       `node $SCRIPTS_DIR/generateCategoryGroupOrder.js \
         --inputDir "$SRC_DIR/common/config/wv.json/categories/" \
-        --outputDir "$SRC_DIR/common/config/wv.json/"`
+        --outputDir "$SRC_DIR/common/config/wv.json/" \
+        --mode "$MODE"`
   fi
 
   if [ -e "$OPT_DIR/$OPT_SUBDIR/layer-metadata/all.json" ] ; then
@@ -138,24 +175,39 @@ else
   done
 
   # Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json
+  if (( $MODE = "verbose" )) ; then
+    echo "Run mergeConfigWithWMTS.js to merge layer metadata from WMTS GC with worldview layer configs into wv.json"
+  fi
   `node $SCRIPTS_DIR/mergeConfigWithWMTS.js \
     --inputDir "$BUILD_DIR/_wmts" \
-    --outputFile "$DEST_DIR/config/wv.json"`
+    --outputFile "$DEST_DIR/config/wv.json" \
+    --mode "$MODE"`
 
   # Copy brand files from build to dest
+  if (( $MODE = "verbose" )) ; then
+    echo "Copy brand files from build to dest"
+  fi
   cp -r "$BUILD_DIR/brand" "$DEST_DIR"
   cp "$BUILD_DIR/brand.json" "$DEST_DIR"
 
 
   # Validate the options build
+  if (( $MODE = "verbose" )) ; then
+    echo "Validate the options build"
+  fi
   `node $SCRIPTS_DIR/validateOptions.js \
     --optionsFile "$BUILD_DIR/config.json" \
-    --configDir "$DEST_DIR/config"`
+    --configDir "$DEST_DIR/config" \
+    --mode "$MODE"`
 
   # Fetch preview images from WV Snapshots for any layers which they are missing
+  if (( $MODE = "verbose" )) ; then
+    echo "Fetch preview images from WV Snapshots for any layers which they are missing"
+  fi
   `node $SCRIPTS_DIR/fetchPreviewSnapshots.js \
     --wvJsonFile "$DEST_DIR/config/wv.json" \
     --overridesFile "$OPT_DIR/common/previewLayerOverrides.json" \
-    --featuresFile "$BUILD_DIR/features.json"`
+    --featuresFile "$BUILD_DIR/features.json" \
+    --mode "$MODE"`
 fi
 exit 0