From 48590c6a484ee06fc7617e7c2a4bebbfdf35028c Mon Sep 17 00:00:00 2001 From: Hossein Firooz Date: Tue, 19 Nov 2024 13:45:03 +0200 Subject: [PATCH] Allow audio file/folder names with whitespaces (#32) Fix OOD error with files/folder with spaces #31. This is done by adding a dynamic form for OOD, and better argument parser for the python script. Other modifications: Replace tmp folder from $WRKDIR/.speech2text to OOD session ID #30. This is mainly for OOD users to have all the log in the OOD app. Add logging information to first slurm submission #29. If the file is not supported, the name of the file will be logged for better debugging. Add OOD dev build #26. Now deploying script supports ood-dev to easily install the ood sandbox for testing. Add specific versions for faster-whisper, ctranslate, whisperX, cuda, and pytorch to env.yml for easier environment building. Add a prefix to slurm job_name for easier statistics analysis using slurm database. --- README.md | 1 + bin/deploy | 7 ++- bin/deploy-data/ood/form.yml | 53 ++--------------------- bin/deploy-data/ood/manifest.yml | 3 +- bin/deploy-data/update_ood_form_values.py | 29 +++++++++++++ env.yml | 3 +- 6 files changed, 43 insertions(+), 53 deletions(-) create mode 100644 bin/deploy-data/update_ood_form_values.py diff --git a/README.md b/README.md index 15cbe2f..9638d3e 100644 --- a/README.md +++ b/README.md @@ -273,6 +273,7 @@ To clean up: - purge loaded modules with `module purge` - remove the local development repo folders `0000-0`, `0000-1`, etc. - remove the OOD sandbox app folder `/scratch/work/$USERNAME/.ondemand/dev/speech2text-dev` +- remove the version-specific module file (`0000-0.lua`, `0000-1.lua`, etc.) at `/appl/manual_installations/modules/speech2text/` ## Build and run with Singularity diff --git a/bin/deploy b/bin/deploy index 7f0e934..b89d31d 100755 --- a/bin/deploy +++ b/bin/deploy @@ -107,6 +107,9 @@ if [ "$OOD" = true ] then echo "Deploy the Open OnDemand (OOD) app" + + # Update supported languages and models + python3 bin/deploy-data/update_ood_form_values.py SCRIPT="$SCRIPT_DIR"/deploy-data/ood/template/script.sh @@ -163,6 +166,9 @@ then echo "Deploy the Open OnDemand (OOD) sandbox app" + # Update supported languages and models + python3 bin/deploy-data/update_ood_form_values.py + SCRIPT="$SCRIPT_DIR"/deploy-data/ood/template/script.sh sed -i "s||$VERSION|g" $SCRIPT @@ -176,7 +182,6 @@ then sed -i "s||$MPLCONFIGDIR|g" $SCRIPT sed -i "s||$SPEECH2TEXT_MEM|g" $SCRIPT sed -i "s||$SPEECH2TEXT_CPUS_PER_TASK|g" $SCRIPT - USERNAME=$(whoami) mkdir -p /scratch/work/$USERNAME/.ondemand/dev/speech2text-dev diff --git a/bin/deploy-data/ood/form.yml b/bin/deploy-data/ood/form.yml index 1da906d..d4ecf1a 100644 --- a/bin/deploy-data/ood/form.yml +++ b/bin/deploy-data/ood/form.yml @@ -26,52 +26,8 @@ attributes: display: true help: | Select the language for the audio file(s). - options: - - arabic - - armenian - - bulgarian - - catalan - - chinese - - czech - - danish - - dutch - - english - - estonian - - finnish - - french - - galician - - german - - greek - - hebrew - - hindi - - hungarian - - icelandic - - indonesian - - italian - - japanese - - kazakh - - korean - - latvian - - lithuanian - - malay - - marathi - - nepali - - norwegian - - persian - - polish - - portuguese - - romanian - - russian - - serbian - - slovak - - slovenian - - spanish - - swedish - - thai - - turkish - - ukrainian - - urdu - - vietnamese + options: # Will be filled with bin/deploy-data/update_ood_form_values.py + email_field: label: Email @@ -89,7 +45,4 @@ attributes: display: true help: | (Optional) Select the whisper model. - options: - - [ "default", "large-v3" ] - - [ "v2", "large-v2" ] - - [ "v3", "large-v3" ] \ No newline at end of file + options: # Will be filled with bin/deploy-data/update_ood_form_values.py \ No newline at end of file diff --git a/bin/deploy-data/ood/manifest.yml b/bin/deploy-data/ood/manifest.yml index 7037f20..63d76ed 100644 --- a/bin/deploy-data/ood/manifest.yml +++ b/bin/deploy-data/ood/manifest.yml @@ -4,6 +4,7 @@ category: Interactive Apps subcategory: Speech2text role: batch_connect description: | - This is a form to submit a speech2text job request. + This is a form to submit a speech2text job request. + [Link to the documentation.](https://scicomp.aalto.fi/triton/apps/speech2text/) # metadata: # field_of_science: botany diff --git a/bin/deploy-data/update_ood_form_values.py b/bin/deploy-data/update_ood_form_values.py new file mode 100644 index 0000000..ef6f47b --- /dev/null +++ b/bin/deploy-data/update_ood_form_values.py @@ -0,0 +1,29 @@ +import yaml +import sys +import os + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +from src.settings import (supported_languages, + available_whisper_models, + default_whisper_model) + +# Path to the form.yml file +form_yml_path = 'bin/deploy-data/ood/form.yml' + +# Load the existing form.yml +with open(form_yml_path, 'r') as file: + form_data = yaml.safe_load(file) + +# Update the language_field options with supported languages +form_data['attributes']['language_field']['options'] = list(supported_languages.keys()) + +# Update the model_selector options with available whisper models +form_data['attributes']['model_selector']['options'] = [ + ["default", default_whisper_model] +] + [[model, model] for model in available_whisper_models] + +# Write the updated data back to form.yml +with open(form_yml_path, 'w') as file: + yaml.dump(form_data, file, default_flow_style=False, sort_keys=False) + +print(".. form.yml has been updated with supported languages and Whisper models from settings.py.") diff --git a/env.yml b/env.yml index 56c093d..4738daf 100644 --- a/env.yml +++ b/env.yml @@ -7,6 +7,7 @@ dependencies: - git - ffmpeg - libsndfile + - numpy=1.26.4 - python=3.10 - pydub - pytorch-lightning=2.2.5 @@ -21,4 +22,4 @@ dependencies: # Requires ctranslate 4.4.0 and faster-whisper 1.0.3. Link to the issue: https://github.com/m-bain/whisperX/issues/901 # "https://github.com/m-bain/whisperX/commit/caa7121064c1bb406be30c50891a9b8217252592" or # "https://github.com/m-bain/whisperX/commit/5080b7188c3666b5d8648346c0c12599a58bd695" - - whisperx @ git+https://github.com/federicotorrielli/BetterWhisperX \ No newline at end of file + - whisperx @ git+https://github.com/federicotorrielli/BetterWhisperX