Fixing mypy issues and tests failing

requirements-minimal.txt

@@ -20,7 +20,11 @@ pytimeparse
 pytz
 PyYAML>=5.1
 requests
+scribereader==0.14.1
 task_processing[mesos_executor,k8s]
 Twisted>=19.7.0
 urllib3>=1.24.2
 Werkzeug>=0.15.3
+yelp-clog==7.0.1
+yelp-logging==4.17.0
+yelp-meteorite==2.1.1

requirements.txt

@@ -15,16 +15,17 @@ certifi==2022.12.7
 cffi==1.12.3
 cfn-lint==0.24.4
 chardet==3.0.4
-clusterman-metrics==2.2.1  # used by tron for pre-scaling for Spark runs
 constantly==15.1.0
 cryptography==39.0.1
 dataclasses==0.6
+dateglob==1.1.1
 DateTime==4.3
 decorator==4.4.0
 docker==4.1.0
 ecdsa==0.13.3
 executing==1.2.0
 future==0.18.3
+geogrid==2.1.0
 google-auth==1.23.0
 http-parser==0.9.0
 humanize==0.5.1
@@ -47,11 +48,13 @@ lockfile==0.12.2
 MarkupSafe==2.1.1
 matplotlib-inline==0.1.3
 mock==3.0.5
+monk==3.0.4
 moto==1.3.13
 oauthlib==3.1.0
 parso==0.7.0
 pexpect==4.7.0
 pickleshare==0.7.5
+ply==3.11
 prompt-toolkit==3.0.38
 psutil==5.6.6
 ptyprocess==0.6.0
@@ -78,10 +81,14 @@ rsa==4.9
 s3transfer==0.6.0
 scribereader==0.14.1  # used by tron to get tronjob logs
 setuptools==65.5.1
+simplejson==3.19.2
 six==1.15.0
+srv-configs==1.3.3
 sshpubkeys==3.1.0
 stack-data==0.6.2
 task-processing==0.12.2
+tenacity==8.2.3
+thriftpy2==0.4.17
 traitlets==5.0.0
 Twisted==22.10.0
 typing-extensions==4.5.0
@@ -91,6 +98,7 @@ websocket-client==0.56.0
 Werkzeug==2.2.3
 wrapt==1.11.2
 xmltodict==0.12.0
+yelp-cgeom==1.3.1
 yelp-clog==7.0.1  # scribereader dependency
 yelp-logging==4.17.0  # scribereader dependency
 yelp-meteorite==2.1.1  # used by task-processing to emit metrics, clusterman-metrics dependency

tests/utils/scribereader_test.py

@@ -21,7 +21,7 @@ def test_read_log_stream_for_action_run_min_date_and_max_date_today():
         "tron.utils.scribereader.get_scribereader_host_and_port",
         autospec=True,
         return_value=("host", 1234),
-    ), mock.patch(
+    ), mock.patch("tron.utils.scribereader.scribereader", autospec=True,), mock.patch(
         "tron.utils.scribereader.scribereader.get_stream_reader",
         autospec=True,
     ) as mock_stream_reader, mock.patch(
@@ -97,7 +97,7 @@ def test_read_log_stream_for_action_run_min_date_and_max_date_different_days():
         "tron.utils.scribereader.get_scribereader_host_and_port",
         autospec=True,
         return_value=("host", 1234),
-    ), mock.patch(
+    ), mock.patch("tron.utils.scribereader.scribereader", autospec=True,), mock.patch(
         "tron.utils.scribereader.scribereader.get_stream_reader",
         autospec=True,
     ) as mock_stream_reader, mock.patch(
@@ -195,7 +195,7 @@ def test_read_log_stream_for_action_run_min_date_and_max_date_in_past():
         "tron.utils.scribereader.get_scribereader_host_and_port",
         autospec=True,
         return_value=("host", 1234),
-    ), mock.patch(
+    ), mock.patch("tron.utils.scribereader.scribereader", autospec=True,), mock.patch(
         "tron.utils.scribereader.scribereader.get_stream_reader",
         autospec=True,
     ) as mock_stream_reader, mock.patch(
@@ -261,7 +261,7 @@ def test_read_log_stream_for_action_run_min_date_and_max_date_for_long_output():
         "tron.utils.scribereader.get_scribereader_host_and_port",
         autospec=True,
         return_value=("host", 1234),
-    ), mock.patch(
+    ), mock.patch("tron.utils.scribereader.scribereader", autospec=True,), mock.patch(
         "tron.utils.scribereader.scribereader.get_stream_reader",
         autospec=True,
     ) as mock_stream_reader, mock.patch(

tox.ini

@@ -25,7 +25,7 @@ commands =
     check-requirements
     # optionally install yelpy requirements - this is after check-requirements since
     # check-requirements doesn't understand these extra requirements
-    -pip-custom-platform install --index-url https://pypi.yelpcorp.com/simple -r yelp_package/extra_requirements_yelp.txt
+    -pip install --index-url https://pypi.yelpcorp.com/simple -r yelp_package/extra_requirements_yelp.txt
     # we then run tests at the very end so that we can run tests with yelpy requirements
     py.test -s {posargs:tests}
 

tron/commands/client.py

@@ -18,7 +18,7 @@
 
     assert simplejson  # Pyflakes
 except ImportError:
-    import json as simplejson
+    import json as simplejson  # type: ignore
 
 log = logging.getLogger(__name__)
 
@@ -100,11 +100,11 @@ def build_get_url(url, data=None):
 
 
 def ensure_user_attribution(headers: Mapping[str, str]) -> Dict[str, str]:
-    headers = headers.copy()
+    headers = headers.copy()  # type: ignore
     if "User-Agent" not in headers:
-        headers["User-Agent"] = USER_AGENT
-    headers["User-Agent"] += f' ({os.environ.get("USER", "anonymous")})'
-    return headers
+        headers["User-Agent"] = USER_AGENT  # type: ignore
+    headers["User-Agent"] += f' ({os.environ.get("USER", "anonymous")})'  # type: ignore
+    return headers  # type: ignore
 
 
 class Client:

tron/commands/display.py

@@ -269,7 +269,7 @@ class DisplayJobRuns(TableDisplay):
     colors = {
         "id": partial(Color.set, "yellow"),
         "state": add_color_for_state,
-        "manual": lambda value: Color.set("cyan" if value else None, value),
+        "manual": lambda value: Color.set("cyan" if value else None, value),  # type: ignore
     }
 
     def format_value(self, field_idx, value):

tron/config/config_utils.py

@@ -275,8 +275,8 @@ def build_child_context(self, path):
 
 class NullConfigContext:
     path = ""
-    nodes = set()
-    command_context = {}
+    nodes = set()  # type: ignore
+    command_context = {}  # type: ignore
     namespace = MASTER_NAMESPACE
     partial = False
 
@@ -292,8 +292,8 @@ class Validator:
     """
 
     config_class = None
-    defaults = {}
-    validators = {}
+    defaults = {}  # type: ignore
+    validators = {}  # type: ignore
     optional = False
 
     def validate(self, in_dict, config_context):

tron/config/schema.py

@@ -260,7 +260,7 @@ def config_object_factory(name, required=None, optional=None):
 )
 
 
-class ConfigSecretVolume(_ConfigSecretVolume):
+class ConfigSecretVolume(_ConfigSecretVolume):  # type: ignore
     def _asdict(self) -> dict:
         d = super()._asdict().copy()
         items = d.get("items", [])
@@ -274,7 +274,7 @@ def _asdict(self) -> dict:
             for i, item in enumerate(items):
                 if isinstance(item, ConfigSecretVolumeItem):
                     d["items"][i] = item._asdict()
-        return d
+        return d  # type: ignore
 
 
 ConfigSecretSource = config_object_factory(
@@ -304,15 +304,15 @@ def _asdict(self) -> dict:
     optional=[],
 )
 
-StatePersistenceTypes = Enum(
+StatePersistenceTypes = Enum(  # type: ignore
     "StatePersistenceTypes",
     dict(shelve="shelve", yaml="yaml", dynamodb="dynamodb"),
 )
 
-ExecutorTypes = Enum("ExecutorTypes", dict(ssh="ssh", mesos="mesos", kubernetes="kubernetes", spark="spark"))
+ExecutorTypes = Enum("ExecutorTypes", dict(ssh="ssh", mesos="mesos", kubernetes="kubernetes", spark="spark"))  # type: ignore
 
-ActionRunnerTypes = Enum("ActionRunnerTypes", dict(none="none", subprocess="subprocess"))
+ActionRunnerTypes = Enum("ActionRunnerTypes", dict(none="none", subprocess="subprocess"))  # type: ignore
 
-VolumeModes = Enum("VolumeModes", dict(RO="RO", RW="RW"))
+VolumeModes = Enum("VolumeModes", dict(RO="RO", RW="RW"))  # type: ignore
 
-ActionOnRerun = Enum("ActionOnRerun", dict(rerun="rerun"))
+ActionOnRerun = Enum("ActionOnRerun", dict(rerun="rerun"))  # type: ignore

tron/config/static_config.py

@@ -1,6 +1,6 @@
 from functools import partial
 
-import staticconf
+import staticconf  # type: ignore
 from staticconf import config
 
 FILENAME = "/nail/srv/configs/tron.yaml"

tron/core/action.py

@@ -20,13 +20,13 @@ class ActionCommandConfig:
     """A configurable data object for one try of an Action."""
 
     command: str
-    cpus: float = None
-    mem: float = None
-    disk: float = None
+    cpus: Optional[float] = None
+    mem: Optional[float] = None
+    disk: Optional[float] = None
     cap_add: List[str] = field(default_factory=list)
     cap_drop: List[str] = field(default_factory=list)
     constraints: set = field(default_factory=set)
-    docker_image: str = None
+    docker_image: Optional[str] = None
     # XXX: we can get rid of docker_parameters once we're off of Mesos
     docker_parameters: set = field(default_factory=set)
     env: dict = field(default_factory=dict)
@@ -56,14 +56,14 @@ class Action:
     name: str
     command_config: ActionCommandConfig
     node_pool: str
-    retries: int = None
-    retries_delay: datetime.timedelta = None
-    expected_runtime: datetime.timedelta = None
-    executor: str = None
-    trigger_downstreams: (bool, dict) = None
-    triggered_by: set = None
-    on_upstream_rerun: str = None
-    trigger_timeout: datetime.timedelta = None
+    retries: Optional[int] = None
+    retries_delay: Optional[datetime.timedelta] = None
+    expected_runtime: Optional[datetime.timedelta] = None
+    executor: Optional[str] = None
+    trigger_downstreams: (bool, dict) = None  # type: ignore
+    triggered_by: Optional[set] = None
+    on_upstream_rerun: Optional[str] = None
+    trigger_timeout: Optional[datetime.timedelta] = None
 
     @property
     def is_cleanup(self):