[FIX] Workflow file path for table extractor (#616)

* Fixed file path Signed-off-by: Deepak <[email protected]> * Added sample.env Signed-off-by: Deepak <[email protected]> * Minor fix Signed-off-by: Deepak <[email protected]> * Updated EXECUTION_RUN_DATA_FOLDER_PREFIX Signed-off-by: Deepak <[email protected]> * Addressed review comment Signed-off-by: Deepak <[email protected]> * Update worker/src/unstract/worker/clients/test_docker.py Signed-off-by: Ritwik G <[email protected]> * Update worker/src/unstract/worker/clients/test_docker.py Signed-off-by: Ritwik G <[email protected]> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update worker/src/unstract/worker/clients/docker.py Co-authored-by: Ritwik G <[email protected]> Signed-off-by: Deepak K <[email protected]> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Deepak <[email protected]> Signed-off-by: Ritwik G <[email protected]> Signed-off-by: Deepak K <[email protected]> Co-authored-by: Ritwik G <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Zipstack · Aug 22, 2024 · beeba6a · beeba6a
1 parent a3763c8
commit beeba6a
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 2 deletions.
diff --git a/worker/sample.env b/worker/sample.env
@@ -19,3 +19,5 @@ REMOVE_CONTAINER_ON_EXIT=True
 
 # Client module path of the container engine to be used.
 CONTAINER_CLIENT_PATH=unstract.worker.clients.docker
+
+EXECUTION_RUN_DATA_FOLDER_PREFIX="/app/workflow_data"
diff --git a/worker/src/unstract/worker/clients/docker.py b/worker/src/unstract/worker/clients/docker.py
@@ -181,7 +181,12 @@ def get_container_run_config(
                     "target": os.getenv(Env.TOOL_DATA_DIR, "/data"),
                 }
             )
-            envs[Env.EXECUTION_RUN_DATA_FOLDER] = source_path
+            envs[Env.EXECUTION_RUN_DATA_FOLDER] = os.path.join(
+                os.getenv(Env.EXECUTION_RUN_DATA_FOLDER_PREFIX, ""),
+                organization_id,
+                workflow_id,
+                execution_id,
+            )
         return {
             "name": ContainerClientHelper.normalize_container_name(self.image_name),
             "image": self.get_image(),

diff --git a/worker/src/unstract/worker/clients/test_docker.py b/worker/src/unstract/worker/clients/test_docker.py
@@ -109,6 +109,7 @@ def test_get_image(docker_client, mocker):
 def test_get_container_run_config(docker_client, mocker):
     """Test the get_container_run_config method."""
     os.environ[Env.WORKFLOW_DATA_DIR] = "/source"
+    os.environ[Env.EXECUTION_RUN_DATA_FOLDER_PREFIX] = "/app/workflow_data"
     command = ["echo", "hello"]
     organization_id = "org123"
     workflow_id = "wf123"
@@ -134,7 +135,7 @@ def test_get_container_run_config(docker_client, mocker):
     assert config["command"] == ["echo", "hello"]
     assert config["environment"] == {
         "KEY": "VALUE",
-        "EXECUTION_RUN_DATA_FOLDER": "/source/org123/wf123/ex123",
+        "EXECUTION_RUN_DATA_FOLDER": ("/app/workflow_data/org123/wf123/ex123"),
     }
     assert config["mounts"] == [
         {

diff --git a/worker/src/unstract/worker/constants.py b/worker/src/unstract/worker/constants.py
@@ -22,6 +22,7 @@ class Env:
     TOOL_CONTAINER_LABELS = "TOOL_CONTAINER_LABELS"
     WORKFLOW_DATA_DIR = "WORKFLOW_DATA_DIR"
     EXECUTION_RUN_DATA_FOLDER = "EXECUTION_RUN_DATA_FOLDER"
+    EXECUTION_RUN_DATA_FOLDER_PREFIX = "EXECUTION_RUN_DATA_FOLDER_PREFIX"
     TOOL_DATA_DIR = "TOOL_DATA_DIR"
     PRIVATE_REGISTRY_CREDENTIAL_PATH = "PRIVATE_REGISTRY_CREDENTIAL_PATH"
     PRIVATE_REGISTRY_USERNAME = "PRIVATE_REGISTRY_USERNAME"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -19,3 +19,5 @@ REMOVE_CONTAINER_ON_EXIT=True

		# Client module path of the container engine to be used.
		CONTAINER_CLIENT_PATH=unstract.worker.clients.docker

		EXECUTION_RUN_DATA_FOLDER_PREFIX="/app/workflow_data"