refactor: DSLRunner now uses a pull-based execution model (#273)

* partial commit

* refactor: dslrunner is now a pull based execution model

* refactor: dslrunner is now a pull based execution model

* refactor: optimizer is now using the new pull based execution model

* refactor: optimizer is now using the new pull based execution model

* refactor: optimizer is now using the new pull based execution model

* remove builder file

* remove builder file and make tests pass

* fix tests

.pre-commit-config.yaml

@@ -20,10 +20,10 @@ repos:
       - id: check-docstring-first
 
   - repo: https://github.com/hadialqattan/pycln
-    rev: v2.4.0
+    rev: v2.5.0
     hooks:
       - id: pycln
-        args: [--all]
+        args: [--all, --exclude, "__init__.py$", --include, "^docetl/"]
 
   - repo: https://github.com/psf/black
     rev: 24.1.1

docetl/__init__.py

@@ -1,6 +1,6 @@
 __version__ = "0.2.1"
 
 from docetl.runner import DSLRunner
-from docetl.builder import Optimizer
+from docetl.optimizer import Optimizer
 
 __all__ = ["DSLRunner", "Optimizer"]

docetl/api.py

@@ -49,32 +49,31 @@
     result = optimized_pipeline.run()
 """
 
-import os
 import inspect
-from typing import Any, Dict, List, Optional, Callable, Union
+import os
+from typing import Any, Callable, Dict, List, Optional, Union
 
 import yaml
 from rich import print
 
-from docetl.builder import Optimizer
 from docetl.runner import DSLRunner
 from docetl.schemas import (
+    ClusterOp,
     Dataset,
     EquijoinOp,
     FilterOp,
     GatherOp,
     MapOp,
-    ReduceOp,
-    ResolveOp,
-    SplitOp,
-    UnnestOp,
-    ClusterOp,
-    SampleOp,
     OpType,
     ParallelMapOp,
     ParsingTool,
     PipelineOutput,
     PipelineStep,
+    ReduceOp,
+    ResolveOp,
+    SampleOp,
+    SplitOp,
+    UnnestOp,
 )
 
 
@@ -166,9 +165,10 @@ def __init__(
         self._load_env()
 
     def _load_env(self):
-        from dotenv import load_dotenv
         import os
 
+        from dotenv import load_dotenv
+
         # Get the current working directory
         cwd = os.getcwd()
 

docetl/base_schemas.py

@@ -1,10 +1,11 @@
 from typing import Any, Dict, List, Optional, Union
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel
 
 # from ..operations import map
 # MapOp = map.MapOperation.schema
 
+
 class ToolFunction(BaseModel):
     name: str
     description: str
@@ -96,7 +97,8 @@ class PipelineStep(BaseModel):
     name: str
     operations: List[Union[Dict[str, Any], str]]
     input: Optional[str] = None
-
+
+
 class PipelineOutput(BaseModel):
     """
     Represents the output configuration for a pipeline.