diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index b0bb413a02b..f12caf2b710 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -401,6 +401,10 @@ OTEL_ENDPOINT = from_conf("OTEL_ENDPOINT") ZIPKIN_ENDPOINT = from_conf("ZIPKIN_ENDPOINT") CONSOLE_TRACE_ENABLED = from_conf("CONSOLE_TRACE_ENABLED", False) +# internal env used for preventing the tracing module from loading during Conda bootstrapping. +BOOTSTRAPPING_CONDA_ENVIRONMENT = bool( + os.environ.get("BOOTSTRAPPING_CONDA_ENVIRONMENT", False) +) # MAX_ATTEMPTS is the maximum number of attempts, including the first # task, retries, and the final fallback task and its retries. diff --git a/metaflow/plugins/conda/conda_environment.py b/metaflow/plugins/conda/conda_environment.py index cd0cb0a65aa..ef4638ff074 100644 --- a/metaflow/plugins/conda/conda_environment.py +++ b/metaflow/plugins/conda/conda_environment.py @@ -76,7 +76,11 @@ def bootstrap_commands(self, step_name, datastore_type): if env_id is not None: return [ "echo 'Bootstrapping environment...'", - 'python -m metaflow.plugins.conda.batch_bootstrap "%s" %s "%s"' + # We have to prevent the tracing module from loading, + # as the bootstrapping process uses the internal S3 client which would fail to import tracing + # due to the required dependencies being bundled into the conda environment, + # which is yet to be initialized at this point. + 'BOOTSTRAPPING_CONDA_ENVIRONMENT=True python -m metaflow.plugins.conda.batch_bootstrap "%s" %s "%s"' % (self.flow.name, env_id, datastore_type), "echo 'Environment bootstrapped.'", ] diff --git a/metaflow/tracing/__init__.py b/metaflow/tracing/__init__.py index 72da6682d0f..aadb7793c6c 100644 --- a/metaflow/tracing/__init__.py +++ b/metaflow/tracing/__init__.py @@ -2,6 +2,7 @@ OTEL_ENDPOINT, ZIPKIN_ENDPOINT, CONSOLE_TRACE_ENABLED, + BOOTSTRAPPING_CONDA_ENVIRONMENT, ) from functools import wraps import contextlib @@ -49,7 +50,9 @@ def wrapper_func(*args, **kwargs): return wrapper_func -if CONSOLE_TRACE_ENABLED or OTEL_ENDPOINT or ZIPKIN_ENDPOINT: +if not BOOTSTRAPPING_CONDA_ENVIRONMENT and ( + CONSOLE_TRACE_ENABLED or OTEL_ENDPOINT or ZIPKIN_ENDPOINT +): try: from .tracing_modules import *