feat(iast): include as iast telemetry metrics tags in the root span o…

…f the trace (#7354)

For metrics that need to be correlated to traces, a small subset of IAST
telemetry metrics can be included as tags in the root span of the trace
following the next algorithm, this PR contains:
- `iast_metrics` into asm_request_context
- Span metrics with IAST `request.tainted` and `executed.skins.*` data

TODO: add integration tests with a flask server and retrieve telemetry
events from test agent but we need to refactor telemetry tests and move
testagent fixture from conftest to a upper level conftest


## Checklist

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed. If no release note is required, add label
`changelog/no-changelog`.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist

- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
- [x] If this PR touches code that signs or publishes builds or
packages, or handles credentials of any kind, I've requested a review
from `@DataDog/security-design-and-guidance`.
- [x] This PR doesn't touch any of that.

---------

Co-authored-by: Christophe Papazian <[email protected]>

ddtrace/appsec/_constants.py

@@ -83,6 +83,13 @@ class IAST(metaclass=Constant_Class):
     SEP_MODULES = ","
 
 
+class IAST_SPAN_TAGS(metaclass=Constant_Class):
+    """Specific constants for IAST span tags"""
+
+    TELEMETRY_REQUEST_TAINTED = "_dd.iast.telemetry.request.tainted"
+    TELEMETRY_EXECUTED_SINK = "_dd.iast.telemetry.executed.sink"
+
+
 class WAF_DATA_NAMES(metaclass=Constant_Class):
     """string names used by the waf library for requesting data from requests"""
 

ddtrace/appsec/_iast/_metrics.py

@@ -1,6 +1,8 @@
 import os
+from typing import Dict
 
 from ddtrace.appsec._constants import IAST
+from ddtrace.appsec._constants import IAST_SPAN_TAGS
 from ddtrace.appsec._deduplications import deduplication
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.telemetry import telemetry_writer
@@ -26,6 +28,8 @@
     (TELEMETRY_OFF_VERBOSITY, TELEMETRY_OFF_NAME),
 )
 
+_IAST_SPAN_METRICS: Dict[str, int] = {}
+
 
 def get_iast_metrics_report_lvl(*args, **kwargs):
     report_lvl_name = os.environ.get(IAST.TELEMETRY_REPORT_LVL, TELEMETRY_INFORMATION_NAME).upper()
@@ -98,10 +102,47 @@ def _set_metric_iast_executed_sink(vulnerability_type):
     )
 
 
-@metric_verbosity(TELEMETRY_INFORMATION_VERBOSITY)
-def _set_metric_iast_request_tainted():
+def _request_tainted():
     from ._taint_tracking import num_objects_tainted
 
-    total_objects_tainted = num_objects_tainted()
+    return num_objects_tainted()
+
+
+@metric_verbosity(TELEMETRY_INFORMATION_VERBOSITY)
+def _set_metric_iast_request_tainted():
+    total_objects_tainted = _request_tainted()
     if total_objects_tainted > 0:
         telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_IAST, "request.tainted", total_objects_tainted)
+
+
+def _set_span_tag_iast_request_tainted(span):
+    total_objects_tainted = _request_tainted()
+
+    if total_objects_tainted > 0:
+        span.set_tag(IAST_SPAN_TAGS.TELEMETRY_REQUEST_TAINTED, total_objects_tainted)
+
+
+def _set_span_tag_iast_executed_sink(span):
+    data = get_iast_span_metrics()
+
+    if data is not None:
+        for key, value in data.items():
+            if key.startswith(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK):
+                span.set_tag(key, value)
+
+    reset_iast_span_metrics()
+
+
+def increment_iast_span_metric(prefix: str, metric_key: str, counter: int = 1) -> None:
+    data = get_iast_span_metrics()
+    full_key = prefix + "." + metric_key.lower()
+    result = data.get(full_key, 0)
+    data[full_key] = result + counter
+
+
+def get_iast_span_metrics() -> Dict:
+    return _IAST_SPAN_METRICS
+
+
+def reset_iast_span_metrics() -> None:
+    _IAST_SPAN_METRICS.clear()

ddtrace/appsec/_iast/processor.py

@@ -13,6 +13,8 @@
 from .._trace_utils import _asm_manual_keep
 from . import oce
 from ._metrics import _set_metric_iast_request_tainted
+from ._metrics import _set_span_tag_iast_executed_sink
+from ._metrics import _set_span_tag_iast_request_tainted
 from ._utils import _iast_report_to_str
 from ._utils import _is_iast_enabled
 
@@ -57,13 +59,12 @@ def on_span_finish(self, span):
         data = core.get_item(IAST.CONTEXT_KEY, span=span)
 
         if data:
-            span.set_tag_str(
-                IAST.JSON,
-                _iast_report_to_str(data),
-            )
+            span.set_tag_str(IAST.JSON, _iast_report_to_str(data))
             _asm_manual_keep(span)
 
         _set_metric_iast_request_tainted()
+        _set_span_tag_iast_request_tainted(span)
+        _set_span_tag_iast_executed_sink(span)
         reset_context()
 
         if span.get_tag(ORIGIN_KEY) is None:

ddtrace/appsec/_iast/taint_sinks/ast_taint.py

@@ -1,6 +1,8 @@
 from typing import TYPE_CHECKING
 
+from ..._constants import IAST_SPAN_TAGS
 from .._metrics import _set_metric_iast_executed_sink
+from .._metrics import increment_iast_span_metric
 from ..constants import DEFAULT_PATH_TRAVERSAL_FUNCTIONS
 from ..constants import DEFAULT_WEAK_RANDOMNESS_FUNCTIONS
 from .path_traversal import check_and_report_path_traversal
@@ -29,6 +31,7 @@ def ast_funcion(
 
     if cls.__class__.__module__ == "random" and cls_name == "Random" and func_name in DEFAULT_WEAK_RANDOMNESS_FUNCTIONS:
         # Weak, run the analyzer
+        increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, WeakRandomness.vulnerability_type)
         _set_metric_iast_executed_sink(WeakRandomness.vulnerability_type)
         WeakRandomness.report(evidence_value=cls_name + "." + func_name)
     elif hasattr(func, "__module__") and DEFAULT_PATH_TRAVERSAL_FUNCTIONS.get(func.__module__):

ddtrace/appsec/_iast/taint_sinks/command_injection.py

@@ -13,7 +13,9 @@
 from ddtrace.internal.logger import get_logger
 from ddtrace.settings.asm import config as asm_config
 
+from ..._constants import IAST_SPAN_TAGS
 from .. import oce
+from .._metrics import increment_iast_span_metric
 from .._utils import _has_to_scrub
 from .._utils import _scrub
 from .._utils import _scrub_get_tokens_positions
@@ -252,5 +254,6 @@ def _iast_report_cmdi(shell_args):
         report_cmdi = shell_args
 
     if report_cmdi:
+        increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, CommandInjection.vulnerability_type)
         _set_metric_iast_executed_sink(CommandInjection.vulnerability_type)
         CommandInjection.report(evidence_value=report_cmdi)

ddtrace/appsec/_iast/taint_sinks/insecure_cookie.py

@@ -2,8 +2,10 @@
 
 from ddtrace.internal.compat import six
 
+from ..._constants import IAST_SPAN_TAGS
 from .. import oce
 from .._metrics import _set_metric_iast_executed_sink
+from .._metrics import increment_iast_span_metric
 from ..constants import EVIDENCE_COOKIE
 from ..constants import VULN_INSECURE_COOKIE
 from ..constants import VULN_NO_HTTPONLY_COOKIE
@@ -47,11 +49,13 @@ def asm_check_cookies(cookies):  # type: (Optional[Dict[str, str]]) -> None
         evidence = "%s=%s" % (cookie_key, cookie_value)
 
         if ";secure" not in lvalue:
+            increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, InsecureCookie.vulnerability_type)
             _set_metric_iast_executed_sink(InsecureCookie.vulnerability_type)
             InsecureCookie.report(evidence_value=evidence)
             return
 
         if ";httponly" not in lvalue:
+            increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, NoHttpOnlyCookie.vulnerability_type)
             _set_metric_iast_executed_sink(NoHttpOnlyCookie.vulnerability_type)
             NoHttpOnlyCookie.report(evidence_value=evidence)
             return
@@ -68,5 +72,6 @@ def asm_check_cookies(cookies):  # type: (Optional[Dict[str, str]]) -> None
             report_samesite = True
 
         if report_samesite:
-            NoSameSite.report(evidence_value=evidence)
+            increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, NoSameSite.vulnerability_type)
             _set_metric_iast_executed_sink(NoSameSite.vulnerability_type)
+            NoSameSite.report(evidence_value=evidence)

ddtrace/appsec/_iast/taint_sinks/path_traversal.py

@@ -2,8 +2,10 @@
 
 from ddtrace.internal.logger import get_logger
 
+from ..._constants import IAST_SPAN_TAGS
 from .. import oce
 from .._metrics import _set_metric_iast_instrumented_sink
+from .._metrics import increment_iast_span_metric
 from .._patch import set_and_check_module_is_patched
 from .._patch import set_module_unpatched
 from ..constants import EVIDENCE_PATH_TRAVERSAL
@@ -52,6 +54,7 @@ def check_and_report_path_traversal(*args: Any, **kwargs: Any) -> None:
             from .._metrics import _set_metric_iast_executed_sink
             from .._taint_tracking import is_pyobject_tainted
 
+            increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, PathTraversal.vulnerability_type)
             _set_metric_iast_executed_sink(PathTraversal.vulnerability_type)
             if is_pyobject_tainted(args[0]):
                 PathTraversal.report(evidence_value=args[0])

ddtrace/appsec/_iast/taint_sinks/ssrf.py

@@ -6,7 +6,9 @@
 from ddtrace.internal.logger import get_logger
 from ddtrace.settings.asm import config as asm_config
 
+from ..._constants import IAST_SPAN_TAGS
 from .. import oce
+from .._metrics import increment_iast_span_metric
 from .._taint_tracking import taint_ranges_as_evidence_info
 from .._utils import _has_to_scrub
 from .._utils import _scrub
@@ -159,6 +161,7 @@ def _iast_report_ssrf(func: Callable, *args, **kwargs):
     from .._metrics import _set_metric_iast_executed_sink
 
     report_ssrf = kwargs.get("url", False)
+    increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, SSRF.vulnerability_type)
     _set_metric_iast_executed_sink(SSRF.vulnerability_type)
     if report_ssrf:
         if oce.request_has_quota and SSRF.has_quota():

ddtrace/appsec/_iast/taint_sinks/weak_cipher.py

@@ -3,9 +3,11 @@
 
 from ddtrace.internal.logger import get_logger
 
+from ..._constants import IAST_SPAN_TAGS
 from .. import oce
 from .._metrics import _set_metric_iast_executed_sink
 from .._metrics import _set_metric_iast_instrumented_sink
+from .._metrics import increment_iast_span_metric
 from .._patch import set_and_check_module_is_patched
 from .._patch import set_module_unpatched
 from .._patch import try_unwrap
@@ -129,6 +131,7 @@ def wrapped_aux_blowfish_function(wrapped, instance, args, kwargs):
 @WeakCipher.wrap
 def wrapped_rc4_function(wrapped, instance, args, kwargs):
     # type: (Callable, Any, Any, Any) -> Any
+    increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, WeakCipher.vulnerability_type)
     _set_metric_iast_executed_sink(WeakCipher.vulnerability_type)
     WeakCipher.report(
         evidence_value="RC4",
@@ -141,6 +144,7 @@ def wrapped_function(wrapped, instance, args, kwargs):
     # type: (Callable, Any, Any, Any) -> Any
     if hasattr(instance, "_dd_weakcipher_algorithm"):
         evidence = instance._dd_weakcipher_algorithm + "_" + str(instance.__class__.__name__)
+        increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, WeakCipher.vulnerability_type)
         _set_metric_iast_executed_sink(WeakCipher.vulnerability_type)
         WeakCipher.report(
             evidence_value=evidence,
@@ -154,6 +158,7 @@ def wrapped_cryptography_function(wrapped, instance, args, kwargs):
     # type: (Callable, Any, Any, Any) -> Any
     algorithm_name = instance.algorithm.name.lower()
     if algorithm_name in get_weak_cipher_algorithms():
+        increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, WeakCipher.vulnerability_type)
         _set_metric_iast_executed_sink(WeakCipher.vulnerability_type)
         WeakCipher.report(
             evidence_value=algorithm_name,

ddtrace/appsec/_iast/taint_sinks/weak_hash.py

@@ -4,9 +4,11 @@
 
 from ddtrace.internal.logger import get_logger
 
+from ..._constants import IAST_SPAN_TAGS
 from .. import oce
 from .._metrics import _set_metric_iast_executed_sink
 from .._metrics import _set_metric_iast_instrumented_sink
+from .._metrics import increment_iast_span_metric
 from .._patch import set_and_check_module_is_patched
 from .._patch import set_module_unpatched
 from .._patch import try_unwrap
@@ -127,6 +129,7 @@ def patch():
 def wrapped_digest_function(wrapped, instance, args, kwargs):
     # type: (Callable, Any, Any, Any) -> Any
     if instance.name.lower() in get_weak_hash_algorithms():
+        increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, WeakHash.vulnerability_type)
         _set_metric_iast_executed_sink(WeakHash.vulnerability_type)
         WeakHash.report(
             evidence_value=instance.name,
@@ -150,6 +153,7 @@ def wrapped_sha1_function(wrapped, instance, args, kwargs):
 def wrapped_new_function(wrapped, instance, args, kwargs):
     # type: (Callable, Any, Any, Any) -> Any
     if args[0].lower() in get_weak_hash_algorithms():
+        increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, WeakHash.vulnerability_type)
         _set_metric_iast_executed_sink(WeakHash.vulnerability_type)
         WeakHash.report(
             evidence_value=args[0].lower(),
@@ -159,6 +163,7 @@ def wrapped_new_function(wrapped, instance, args, kwargs):
 
 def wrapped_function(wrapped, evidence, instance, args, kwargs):
     # type: (Callable, str, Any, Any, Any) -> Any
+    increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, WeakHash.vulnerability_type)
     _set_metric_iast_executed_sink(WeakHash.vulnerability_type)
     WeakHash.report(
         evidence_value=evidence,

ddtrace/contrib/dbapi/__init__.py

@@ -8,6 +8,8 @@
 from ddtrace.appsec._iast._utils import _is_iast_enabled
 from ddtrace.internal.constants import COMPONENT
 
+from ...appsec._constants import IAST_SPAN_TAGS
+from ...appsec._iast._metrics import increment_iast_span_metric
 from ...constants import ANALYTICS_SAMPLE_RATE_KEY
 from ...constants import SPAN_KIND
 from ...constants import SPAN_MEASURED_KEY
@@ -114,6 +116,7 @@ def _trace_method(self, method, name, resource, extra_tags, dbm_propagator, *arg
                     from ddtrace.appsec._iast._taint_utils import check_tainted_args
                     from ddtrace.appsec._iast.taint_sinks.sql_injection import SqlInjection
 
+                    increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, SqlInjection.vulnerability_type)
                     _set_metric_iast_executed_sink(SqlInjection.vulnerability_type)
                     if check_tainted_args(args, kwargs, pin.tracer, self._self_config.integration_name, method):
                         SqlInjection.report(evidence_value=args[0])

ddtrace/contrib/dbapi_async/__init__.py

@@ -2,6 +2,8 @@
 from ddtrace.appsec._iast._utils import _is_iast_enabled
 from ddtrace.internal.constants import COMPONENT
 
+from ...appsec._constants import IAST_SPAN_TAGS
+from ...appsec._iast._metrics import increment_iast_span_metric
 from ...constants import ANALYTICS_SAMPLE_RATE_KEY
 from ...constants import SPAN_KIND
 from ...constants import SPAN_MEASURED_KEY
@@ -77,6 +79,7 @@ async def _trace_method(self, method, name, resource, extra_tags, dbm_propagator
                 from ddtrace.appsec._iast._taint_utils import check_tainted_args
                 from ddtrace.appsec._iast.taint_sinks.sql_injection import SqlInjection
 
+                increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, SqlInjection.vulnerability_type)
                 _set_metric_iast_executed_sink(SqlInjection.vulnerability_type)
                 if check_tainted_args(args, kwargs, pin.tracer, self._self_config.integration_name, method):
                     SqlInjection.report(evidence_value=args[0])

tests/.suitespec.json

@@ -394,8 +394,7 @@
             "@tracing",
             "@appsec",
             "@appsec_iast",
-            "tests/appsec/appsec/*",
-            "tests/snapshots/tests.appsec.*"
+            "tests/appsec/*"
         ],
         "appsec_iast": [
             "@bootstrap",
@@ -411,7 +410,9 @@
             "@tracing",
             "@appsec",
             "@appsec_iast",
-            "tests/appsec/iast_integrations/*"
+            "tests/appsec/*",
+            "tests/appsec/iast_integrations/*",
+            "tests/snapshots/tests.appsec.*"
         ],
         "aws_lambda": [
             "@bootstrap",

tests/appsec/appsec_utils.py

@@ -36,28 +36,38 @@ def gunicorn_server(appsec_enabled="true", remote_configuration_enabled="true",
 
 
 @contextmanager
-def flask_server(appsec_enabled="true", remote_configuration_enabled="true", tracer_enabled="true", token=None):
+def flask_server(
+    appsec_enabled="true", remote_configuration_enabled="true", iast_enabled="false", tracer_enabled="true", token=None
+):
     cmd = ["python", "tests/appsec/integrations/app.py", "--no-reload"]
     yield from appsec_application_server(
         cmd,
         appsec_enabled=appsec_enabled,
         remote_configuration_enabled=remote_configuration_enabled,
+        iast_enabled=iast_enabled,
         tracer_enabled=tracer_enabled,
         token=token,
     )
 
 
 def appsec_application_server(
-    cmd, appsec_enabled="true", remote_configuration_enabled="true", tracer_enabled="true", token=None
+    cmd,
+    appsec_enabled="true",
+    remote_configuration_enabled="true",
+    iast_enabled="false",
+    tracer_enabled="true",
+    token=None,
 ):
     env = _build_env()
     env["DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS"] = "0.5"
     env["DD_REMOTE_CONFIGURATION_ENABLED"] = remote_configuration_enabled
     if token:
         env["_DD_REMOTE_CONFIGURATION_ADDITIONAL_HEADERS"] = "X-Datadog-Test-Session-Token:%s," % (token,)
-    if appsec_enabled:
+    if appsec_enabled is not None:
         env["DD_APPSEC_ENABLED"] = appsec_enabled
-    if tracer_enabled:
+    if iast_enabled is not None and iast_enabled != "false":
+        env["DD_IAST_ENABLED"] = iast_enabled
+    if tracer_enabled is not None:
         env["DD_TRACE_ENABLED"] = tracer_enabled
     env["DD_TRACE_AGENT_URL"] = os.environ.get("DD_TRACE_AGENT_URL", "")