feat: Interruption and termination signals in taps and targets

Author: edgarrmondragon

samples/sample_tap_countries/countries_streams.py

@@ -17,6 +17,9 @@
 from singer_sdk import typing as th
 from singer_sdk.streams.graphql import GraphQLStream
 
+if t.TYPE_CHECKING:
+    from collections.abc import Iterable
+
 SCHEMAS_DIR = importlib.resources.files(__package__) / "schemas"
 
 
@@ -83,6 +86,18 @@ class CountriesStream(CountriesAPIStream):
         ),
     ).to_dict()
 
+    # FIXME: revert these changes before merging
+    def request_records(self, context) -> Iterable[dict]:
+        import time  # noqa: PLC0415
+
+        records = super().request_records(context)
+
+        yield next(records)  # Emit the first record
+
+        time.sleep(60)  # Simulate a slow stream
+
+        yield from records  # Emit the rest of the records
+
 
 class ContinentsStream(CountriesAPIStream):
     """Continents stream from the Countries API."""

samples/sample_target_csv/csv_target.py

@@ -13,7 +13,10 @@ class SampleTargetCSV(Target):
     name = "target-csv"
     config_jsonschema = th.PropertiesList(
         th.Property(
-            "target_folder", th.StringType, required=True, title="Target Folder"
+            "target_folder",
+            th.StringType,
+            default="output",
+            title="Target Folder",
         ),
         th.Property("file_naming_scheme", th.StringType, title="File Naming Scheme"),
     ).to_dict()

singer_sdk/plugin_base.py

@@ -5,13 +5,14 @@
 import abc
 import logging
 import os
+import signal
 import sys
 import time
 import typing as t
 import warnings
 from importlib import metadata
 from pathlib import Path, PurePath
-from types import MappingProxyType
+from types import FrameType, MappingProxyType
 
 import click
 
@@ -217,6 +218,10 @@ def __init__(
         # Initialization timestamp
         self.__initialized_at = int(time.time() * 1000)
 
+        # Signal handling
+        signal.signal(signal.SIGINT, self._handle_termination)
+        signal.signal(signal.SIGTERM, self._handle_termination)
+
     def setup_mapper(self) -> None:
         """Initialize the plugin mapper for this tap."""
         self._mapper = PluginMapper(
@@ -445,6 +450,24 @@ def _validate_config(self, *, raise_errors: bool = True) -> list[str]:
 
         return errors
 
+    def _handle_termination(  # pragma: no cover
+        self,
+        signum: int,  # noqa: ARG002
+        frame: FrameType | None,  # noqa: ARG002
+    ) -> None:
+        """Handle termination signal.
+
+        Args:
+            signum: Signal number.
+            frame: Frame.
+
+        Raises:
+            click.Abort: If the termination signal is received.
+        """
+        self.logger.info("Gracefully shutting down...")
+        errmsg = "Received termination signal"
+        raise click.Abort(errmsg)
+
     @classmethod
     def print_version(
         cls: type[PluginBase],

singer_sdk/tap_base.py

@@ -35,6 +35,7 @@
 
 if t.TYPE_CHECKING:
     from pathlib import PurePath
+    from types import FrameType
 
     from singer_sdk.connectors import SQLConnector
     from singer_sdk.mapper import PluginMapper
@@ -498,6 +499,22 @@ def sync_all(self) -> None:
 
     # Command Line Execution
 
+    def _handle_termination(  # pragma: no cover
+        self,
+        signum: int,
+        frame: FrameType | None,
+    ) -> None:
+        """Handle termination signal.
+
+        Args:
+            signum: Signal number.
+            frame: Frame.
+        """
+        # Emit a final state message to ensure the state is written to the output
+        # even if the process is terminated by a signal.
+        self.write_message(StateMessage(value=self.state))
+        super()._handle_termination(signum, frame)
+
     @classmethod
     def invoke(  # type: ignore[override]
         cls: type[Tap],

singer_sdk/target_base.py

@@ -34,6 +34,7 @@
 if t.TYPE_CHECKING:
     from collections.abc import Iterable
     from pathlib import PurePath
+    from types import FrameType
 
     from singer_sdk.connectors import SQLConnector
     from singer_sdk.mapper import PluginMapper
@@ -537,6 +538,24 @@ def _write_state_message(self, state: dict) -> None:
 
     # CLI handler
 
+    def _handle_termination(  # pragma: no cover
+        self,
+        signum: int,
+        frame: FrameType | None,
+    ) -> None:
+        """Handle termination signals.
+
+        Args:
+            signum: Signal number.
+            frame: Frame object.
+        """
+        self.logger.info(
+            "Received termination signal %d, draining all sinks...",
+            signum,
+        )
+        self.drain_all(is_endofpipe=True)
+        super()._handle_termination(signum, frame)
+
     @classmethod
     def invoke(  # type: ignore[override]
         cls: type[Target],