disable proxy for DoQ for now - it sort of works but not consistently…

…, needs more monkeypatching. Add proxy tests using gera2ld.socks, add more debug logging

pyproject.toml

@@ -31,7 +31,7 @@ dns_exporter = "dns_exporter.entrypoint:main"
 
 [project.optional-dependencies]
 dev = ["pre-commit == 3.6.2", "setuptools-scm == 8.0.4"]
-test = ["pytest == 8.0.1", "pytest-cov==4.1.0", "tox == 4.13.0", "requests==2.31.0", "pytest-randomly==3.15.0", "pytest-mock==3.12.0"]
+test = ["pytest == 8.0.1", "pytest-cov==4.1.0", "tox == 4.13.0", "requests==2.31.0", "pytest-randomly==3.15.0", "pytest-mock==3.12.0", "gera2ld.socks==0.5.0"]
 docs = ["Sphinx==7.2.6", "furo==2024.1.29"]
 
 [project.urls]

src/conftest.py

@@ -24,6 +24,8 @@ def dns_exporter_no_main_no_config():
     thread.daemon = True
     thread.start()
     time.sleep(1)
+    if not thread.is_alive():
+        pytest.fail("Unable to create test instance on 127.0.0.1:45353")
     yield
     print("Beginning teardown")
 
@@ -39,6 +41,8 @@ def dns_exporter_example_config():
     thread.daemon = True
     thread.start()
     time.sleep(1)
+    if not thread.is_alive():
+        pytest.fail("Unable to create test instance on 127.0.0.1:25353")
     yield
     print("Beginning teardown")
 
@@ -54,6 +58,8 @@ def dns_exporter_main_no_config_no_debug():
     thread.daemon = True
     thread.start()
     time.sleep(1)
+    if not thread.is_alive():
+        pytest.fail("Unable to create test instance on 127.0.0.1:35353")
     yield
     print("Beginning teardown")
 
@@ -66,10 +72,12 @@ def dns_exporter_param_config(request):
     proc = subprocess.Popen(
         args=["dns_exporter", "-c", str(conf), "-d"],
     )
+    time.sleep(1)
     if proc.poll():
         # process didn't start properly, bail out
-        return
-    time.sleep(1)
+        pytest.fail(
+            f"Unable to create test instance with config {request.param} on 127.0.0.1:15353"
+        )
     yield
     print(f"Stopping dns_exporter with config {request.param} on 127.0.0.1:15353 ...")
     proc.terminate()
@@ -135,9 +143,11 @@ def prometheus_server(request, tmp_path_factory, tmpdir_factory):
             "--storage.tsdb.path",
             prompath,
             "--web.listen-address",
-            "127.0.0.1:9091",
+            "127.0.0.1:9092",
         ],
     )
+    if proc.poll():
+        pytest.fail("Unable to start prometheus on 127.0.0.1:9091")
     print("Setup finished - prometheus is running!")
 
     # end buildup
@@ -147,6 +157,7 @@ def prometheus_server(request, tmp_path_factory, tmpdir_factory):
     print("Beginning teardown")
     print("Stopping prometheus server...")
     proc.terminate()
+    proc.communicate()
     print("Teardown finished!")
 
 
@@ -156,3 +167,31 @@ def mock_collect_httpx_connecterror(mocker):
         "dns_exporter.collector.DNSCollector.get_dns_response",
         side_effect=httpx.ConnectError("mocked"),
     )
+
+
+@pytest.fixture(scope="session")
+def proxy_server():
+    print("Running proxy server on 127.0.0.1:1080...")
+    proc = subprocess.Popen(
+        args=[
+            "python3",
+            "-m",
+            "gera2ld.socks.server",
+            "-b",
+            "127.0.0.1:1080",
+        ],
+    )
+    time.sleep(2)
+    if proc.poll():
+        pytest.fail("Unable to start proxy on 127.0.0.1:1080")
+    print("Setup finished - proxy is running on 127.0.0.1:1080!")
+
+    # end buildup
+    yield
+    # begin teardown
+
+    print("Beginning teardown")
+    print("Stopping proxy server...")
+    proc.terminate()
+    proc.communicate()
+    print("Teardown finished!")

src/dns_exporter/collector.py

@@ -123,7 +123,7 @@ def collect_dns(self) -> Iterator[Union[CounterMetricFamily, GaugeMetricFamily]]
             )
         except Exception:
             logger.exception(
-                f"Caught an unknown exception while looking up qname {self.config.query_name} using server {str(self.config.server.geturl())} - exception details follow, returning other_failure"
+                f"Caught an unknown exception while looking up qname {self.config.query_name} using server {self.config.server.geturl()} and proxy {self.config.proxy.geturl() if self.config.proxy else 'none'} - exception details follow, returning other_failure"
             )
             yield from self.yield_failure_reason_metric(failure_reason="other_failure")
         # clock it

src/dns_exporter/config.py

@@ -266,8 +266,8 @@ def __post_init__(self) -> None:
 
         # validate proxy
         if self.proxy:
-            # proxy support doesn't work for DoT for now
-            if self.protocol in ["dot"]:
+            # proxy support doesn't work for DoT and DoQ for now
+            if self.protocol in ["dot", "doq"]:
                 logger.error(f"proxy not valid for protocol {self.protocol}")
                 raise ConfigError(
                     "invalid_request_config",

src/dns_exporter/exporter.py

@@ -438,7 +438,9 @@ def do_GET(self) -> None:
         """Handle incoming HTTP GET requests."""
         # parse the scrape request url and querystring
         self.url, self.qs = self.parse_querystring()
-
+        logger.debug(
+            f"Got HTTP request for {self.url.geturl()} - parsed qs is {self.qs}"
+        )
         # increase the persistent http request metric
         dnsexp_http_requests_total.labels(path=self.url.path).inc()
 

src/tests/test_examples.py

@@ -25,7 +25,7 @@ def test_list_of_servers(prometheus_server, dns_exporter_param_config):
     """Test the list_of_servers snippets from the docs."""
     for _ in range(15):
         r = requests.get(
-            'http://127.0.0.1:9091/api/v1/query?query=sum(dnsexp_failures_total{job="dnsexp_doh_gmail_mx"})'
+            'http://127.0.0.1:9092/api/v1/query?query=sum(dnsexp_failures_total{job="dnsexp_doh_gmail_mx"})'
         )
         if (
             len(r.json()["data"]["result"]) > 0
@@ -47,7 +47,7 @@ def test_list_of_names(caplog, prometheus_server, dns_exporter_param_config):
     """Test the list_of_names snippets from the docs."""
     for _ in range(15):
         r = requests.get(
-            'http://127.0.0.1:9091/api/v1/query?query=sum(dnsexp_failures_total{job="dnsexp_quad9_mx"})'
+            'http://127.0.0.1:9092/api/v1/query?query=sum(dnsexp_failures_total{job="dnsexp_quad9_mx"})'
         )
         if (
             len(r.json()["data"]["result"]) > 0

src/tests/test_exporter.py

@@ -279,14 +279,14 @@ def test_internal_metrics(dns_exporter_example_config, caplog):
     assert f'dnsexp_build_version_info{{version="{__version__}"}} 1.0' in r.text
     assert "Returning exporter metrics for request to /metrics" in caplog.text
     for metric in """dnsexp_http_requests_total{path="/notfound"} 1.0
-dnsexp_http_requests_total{path="/query"} 40.0
+dnsexp_http_requests_total{path="/query"} 43.0
 dnsexp_http_requests_total{path="/config"} 2.0
 dnsexp_http_requests_total{path="/"} 1.0
 dnsexp_http_requests_total{path="/metrics"} 1.0
 dnsexp_http_responses_total{path="/notfound",response_code="404"} 1.0
-dnsexp_http_responses_total{path="/query",response_code="200"} 40.0
+dnsexp_http_responses_total{path="/query",response_code="200"} 43.0
 dnsexp_http_responses_total{path="/",response_code="200"} 1.0
-dnsexp_dns_queries_total 29.0
+dnsexp_dns_queries_total 32.0
 dnsexp_dns_responsetime_seconds_bucket{additional="0",answer="1",authority="0",family="ipv4",flags="QR RA RD",ip="8.8.4.4",le="0.005",nsid="no_nsid",opcode="QUERY",port="53",protocol="udp",proxy="none",query_name="example.com",query_type="A",rcode="NOERROR",server="udp://dns.google:53",transport="UDP"}
 dnsexp_scrape_failures_total{reason="timeout"} 1.0
 dnsexp_scrape_failures_total{reason="invalid_response_flags"} 6.0
@@ -811,3 +811,51 @@ def test_httpx_connecterror(
         },
     )
     assert 'dnsexp_failures_total{reason="connection_error"} 1.0' in r.text
+
+
+def test_proxy_udp(dns_exporter_example_config, proxy_server):
+    """Test proxy functionality for udp protocol."""
+    r = requests.get(
+        "http://127.0.0.1:25353/query",
+        params={
+            "query_name": "example.com",
+            "server": "dns.google",
+            "family": "ipv4",
+            "protocol": "udp",
+            "proxy": "socks5://127.0.0.1:1080",
+        },
+    )
+    assert 'proxy="socks5://127.0.0.1:1080"' in r.text
+    assert 'server="udp://dns.google:53"' in r.text
+
+
+def test_proxy_tcp(dns_exporter_example_config, proxy_server):
+    """Test proxy functionality for tcp protocol."""
+    r = requests.get(
+        "http://127.0.0.1:25353/query",
+        params={
+            "query_name": "example.com",
+            "server": "dns.google",
+            "family": "ipv4",
+            "protocol": "tcp",
+            "proxy": "socks5://127.0.0.1:1080",
+        },
+    )
+    assert 'proxy="socks5://127.0.0.1:1080"' in r.text
+    assert 'server="tcp://dns.google:53"' in r.text
+
+
+def test_proxy_doh(dns_exporter_example_config, proxy_server):
+    """Test proxy functionality for doh protocol."""
+    r = requests.get(
+        "http://127.0.0.1:25353/query",
+        params={
+            "query_name": "example.com",
+            "server": "dns.google",
+            "family": "ipv4",
+            "protocol": "doh",
+            "proxy": "socks5://127.0.0.1:1080",
+        },
+    )
+    assert 'proxy="socks5://127.0.0.1:1080"' in r.text
+    assert 'server="https://dns.google:443/dns-query"' in r.text

tox.ini

@@ -11,6 +11,7 @@ deps =
     pytest-randomly
     pytest-order
     pytest-mock
+    gera2ld.socks
     -e.[test]
 commands = pytest --cov --cov-report=xml --cov-report=html --cov-config=../tox.ini