Update output naming scheme

Author: bacevicius

Dockerfile

@@ -31,6 +31,7 @@ ENV includeComments=true
 ENV excludeStopwords=true
 ENV useTfidf=true
 ENV numberOfTfidfKeywords="50"
+ENV variant="default"
 
 # Training variables
 ENV train=true

docker-compose.yml

@@ -14,6 +14,7 @@ services:
 
     environment:
       dataset: "codesearchnet"
+      variant: "comments"
     # Preprocessing variables
       preprocess: true
       includeComments: true
@@ -22,9 +23,8 @@ services:
       numberOfTfidfKeywords: "50"
     # Training variables
       train: true
+      # There has to ba an existing model for the following to work
       continueTrainingFromCheckpoint: false
-    # Evaluation variables
-    # add more flags here
 
     deploy:
       resources:

entrypoint.sh

@@ -4,12 +4,12 @@
 set -e
 
 if [ "$preprocess" = true ]; 
-then bash preprocess.sh --dataset="$dataset" --include_comments="$includeComments" --exclude_stopwords="$excludeStopwords" --include_tfidf="$useTfidf" --number_keywords="$numberOfTfidfKeywords"
+then bash preprocess.sh --dataset="$dataset" --include_comments="$includeComments" --exclude_stopwords="$excludeStopwords" --include_tfidf="$useTfidf" --number_keywords="$numberOfTfidfKeywords" --variant="$variant"
 else echo "Not preprocessing."
 fi
 
 if [ "$train" = true ]; 
-then bash train.sh --dataset="$dataset" --continue_training_from_checkpoint="$continueTrainingFromCheckpoint"
+then bash train.sh --dataset="$dataset" --continue_training_from_checkpoint="$continueTrainingFromCheckpoint" --variant="$variant"
 else echo "Not training a new model."
 fi
 

preprocess.sh

@@ -27,6 +27,7 @@ set -e
 
 # Default preprocessing values
 DATASET_NAME=default
+VARIANT=default
 INCLUDE_COMMENTS=true
 EXCLUDE_STOPWORDS=false
 USE_TFIDF=false
@@ -49,13 +50,15 @@ while getopts ab:c:-: OPT; do
     exclude_stopwords ) EXCLUDE_STOPWORDS="$OPTARG" ;;
     include_tfidf )  USE_TFIDF="$OPTARG" ;;
     number_keywords )  NUMBER_OF_TFIDF_KEYWORDS="$OPTARG" ;;
+    variant ) VARIANT="$OPTARG" ;;
     ??* )          die "Illegal option --$OPT" ;;  # bad long option
     ? )            exit 2 ;;  # bad short option (error reported via getopts)
   esac
 done
 shift $((OPTIND-1)) # remove parsed options and args from $@ list
 
 echo "Dataset: $DATASET_NAME" 
+echo "Variant: $VARIANT"
 echo "Including comments: $INCLUDE_COMMENTS" 
 echo "Excluding stopwords: $EXCLUDE_STOPWORDS" 
 echo "Using TFIDF: $USE_TFIDF" 
@@ -76,9 +79,9 @@ NUM_THREADS=64
 PYTHON=python3
 ###########################################################
 
-OUTPUT_DIR=${INPUT_DIR}/${DATASET_NAME}/preprocessed
+OUTPUT_DIR=${INPUT_DIR}/${DATASET_NAME}/preprocessed/exp_${VARIANT}
 
-mkdir -p ${INPUT_DIR}/${DATASET_NAME}/preprocessed
+mkdir -p ${INPUT_DIR}/${DATASET_NAME}/preprocessed/exp_${VARIANT}
 
 TRAIN_DATA_FILE=${OUTPUT_DIR}/${DATASET_NAME}.train.raw.txt
 VAL_DATA_FILE=${OUTPUT_DIR}/${DATASET_NAME}.val.raw.txt

train.sh

@@ -7,6 +7,7 @@
 #   on the final (held-out) test set, change 'val' to 'test'.
 
 dataset_name=default
+variant=default
 continue_training_from_checkpoint=true
 
 # This code block is used to get long two-dash arguments from the command line.
@@ -23,6 +24,7 @@ while getopts ab:c:-: OPT; do
   case "$OPT" in
     dataset )    dataset_name="$OPTARG" ;;
     continue_training_from_checkpoint )    continue_training_from_checkpoint="$OPTARG" ;;
+    variant ) variant="$OPTARG" ;;
     ??* )          die "Illegal option --$OPT" ;;  # bad long option
     ? )            exit 2 ;;  # bad short option (error reported via getopts)
   esac
@@ -32,8 +34,8 @@ shift $((OPTIND-1)) # remove parsed options and args from $@ list
 echo "Dataset: $dataset_name" 
 echo "Training from a previous checkpoint: $continue_training_from_checkpoint"
 
-type=trained_${dataset_name}
-data_dir=datasets/${dataset_name}/preprocessed
+type=exp_${dataset_name}_${variant}
+data_dir=datasets/${dataset_name}/preprocessed/exp_${variant}
 data=${data_dir}/${dataset_name}
 test_data=${data_dir}/${dataset_name}.val.c2s
 model_dir=models/${type}