[WIP] First interation of converter

ray-project · Jan 19, 2025 · 0550d6b · 0550d6b
1 parent 3d5149d
commit 0550d6b
Show file tree

Hide file tree

Showing 23 changed files with 1,729 additions and 433 deletions.
diff --git a/deltacat/compute/converter/__init__.py b/deltacat/compute/converter/__init__.py
diff --git a/deltacat/compute/converter/constants.py b/deltacat/compute/converter/constants.py
@@ -0,0 +1,4 @@
+DEFAULT_CONVERTER_TASK_MAX_PARALLELISM = 4096
+
+# Safe limit ONLY considering CPU limit, typically 32 for a 8x-large worker
+DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD = 30