fix: OUTPUT_DIR

Signed-off-by: Nasfame <[email protected]>

Took 50 seconds

Makefile

@@ -15,11 +15,11 @@ clean:
 
 run:
 	# DATA_DIR='$(pwd)/data' poetry run python main.py train_v2 --train_script=linear-regression.ipynb -i=samples/sample_v3/sample_v3.zip
-	DATA_DIR='./data' poetry run python main.py train_v2 --train_script=linear-regression.ipynb -i=samples/sample_v3/sample_v3.zip
+	DATA_DIR='./data' OUTPUT_DIR='./outputs' poetry run python main.py train_v2 --train_script=linear-regression.ipynb -i=samples/sample_v3/sample_v3.zip
 	# DATA_DIR='./data' poetry run python main.py train_v2 -t=linear-regression.ipynb -i=samples/sample_v3/sample_v3.zip
 
 run_cid:
-	DATA_DIR='./data' poetry run python main.py train_v2 -t=linear-regression.ipynb -i=Qme1HnwLHVzRxra7mT5gRkG7WbyE4FhnGFn9inETSj33Hw
+	DATA_DIR='./data'  OUTPUT_DIR='./outputs' poetry run python main.py train_v2 -t=linear-regression.ipynb -i=Qme1HnwLHVzRxra7mT5gRkG7WbyE4FhnGFn9inETSj33Hw
 
 dc:
 	docker-compose down -v --rmi all

docker-compose.yml

@@ -3,14 +3,15 @@ services:
   decenter.train_v2:
     environment:
       - DATA_DIR='/data'
+      - OUTPUT_DIR='/outputs'
     build:
       context: .
       dockerfile: Dockerfile
       args:
         - cmd=train_v2
     volumes:
       - ./data:/data
-      - ./outputs:/app/outputs
+      - ./outputs:/outputs
 
     healthcheck:
       test:

helpers.py

@@ -5,6 +5,8 @@
 import subprocess
 import sys
 
+import zipfile
+
 from icecream import ic
 
 
@@ -101,3 +103,18 @@ def archive_directory(
     ic(created_archive_loc)
 
     return created_archive_loc
+
+
+def archive_directory_custom(output_archive_path, dir_to_archive):
+    with zipfile.ZipFile(
+        output_archive_path,
+        "w",
+        zipfile.ZIP_DEFLATED,
+    ) as zipf:
+        for root, dirs, files in os.walk(dir_to_archive):
+            for file in files:
+                file_path = os.path.join(root, file)
+                zipf.write(
+                    file_path,
+                    os.path.relpath(file_path, dir_to_archive),
+                )

main.py

@@ -46,7 +46,7 @@ def train(train_script: str, requirements_txt: str = None, data_dir=DATA_DIR):
         )
         if path_not_found(requirements_path):
             logging.critical(
-                f"requirements path- {requirements_path} not found"
+                f"requirements path- {requirements_path} not found",
             )
         else:
             install_dependencies(
@@ -88,7 +88,8 @@ def train(train_script: str, requirements_txt: str = None, data_dir=DATA_DIR):
     logging.error(result.stderr)
 
     with open(os.path.join(data_dir, "stdout"), "w") as f1, open(
-        os.path.join(data_dir, "stderr"), "w"
+        os.path.join(data_dir, "stderr"),
+        "w",
     ) as f2:
         f1.write(result.stdout)
         f2.write(result.stderr)
@@ -98,7 +99,9 @@ def train(train_script: str, requirements_txt: str = None, data_dir=DATA_DIR):
 
 
 def train_v2(
-    train_script: str, input_archive: str, requirements_txt: str = None
+    train_script: str,
+    input_archive: str,
+    requirements_txt: str = None,
 ):
     logging.info(f"start {datetime.datetime.utcnow()}")
 
@@ -159,6 +162,6 @@ def train_v2(
 
     logging.basicConfig(level=logging.INFO)
 
-    # fire.Fire(train, 'train', 'Train')
-    # fire.Fire(train_v2, 'train_v2', 'Train v2')
+    # fire.Fire(train, "train", "Train")
+    # fire.Fire(train_v2, "train_v2", "Train v2")
     fire.Fire({"train": train, "train_v2": train_v2})

setup.cfg

@@ -1,4 +1,4 @@
 [flake8]
 per-file-ignores =
     test_*.py: F841, E501, F541, F403, F405, E701, F401, F841, E501
-    *.py: F841, E501, F541, F403, F405, E701, F401, F841, E501
+    *.py: F841, E501, F541, F403, F405, E701, F401, F841, E501, E303