From 2051a2d330a63cd39d3eb44707f86378de1b079a Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 6 Sep 2023 15:15:19 +0200 Subject: [PATCH] cleanup: remove jafar and the previous QA framework (#1244) ## Checklist - [x] I have read the [contribution guidelines](https://github.com/ooni/probe-cli/blob/master/CONTRIBUTING.md) - [x] reference issue for this pull request: https://github.com/ooni/probe/issues/1803 - [x] if you changed anything related to how experiments work and you need to reflect these changes in the ooni/spec repository, please link to the related ooni/spec pull request: N/A - [x] if you changed code inside an experiment, make sure you bump its version number: N/A ## Description This is the final act of the quest to replace Jafar. Let's remove Jafar itself and the QA framework using it. We now have a much better QA framework, that runs for each commit, is faster, and allows to better emulate censorship cases. While working on the removal, I noticed some tutorials depended on a limited set of Jafar functionality, namely the possibility of provoking simple censorship conditions. So, I refactored the previous code for managing iptables on Linux used by Jafar to produce a much slimmer, fully unit tested tool implementing a subset of the original CLI. By implementing this change, we make sure that we still have simple ways for people to learn while reading tutorials. What remains to be done at this point is to update existing issues, close done issues, and generally make sure we explain clearly what we achieved by working on this quest, and what new features are now available. --- .github/workflows/jafar.yml | 26 -- .github/workflows/qa.yml | 15 - .gitignore | 2 +- CONTRIBUTING.md | 7 +- QA/.dockerignore | 1 - QA/.gitignore | 4 - QA/README.md | 24 -- QA/common.py | 68 ---- QA/dockermain.sh | 21 -- QA/minioonilike.py | 88 ----- QA/rundocker.bash | 16 - QA/webconnectivity.py | 101 ----- internal/cmd/jafar/.gitignore | 1 - internal/cmd/jafar/README.md | 257 ------------- internal/cmd/jafar/badproxy/badproxy.go | 114 ------ internal/cmd/jafar/badproxy/badproxy_test.go | 157 -------- internal/cmd/jafar/httpproxy/httpproxy.go | 78 ---- .../cmd/jafar/httpproxy/httpproxy_test.go | 130 ------- internal/cmd/jafar/iptables/iptables.go | 98 ----- .../iptables/iptables_integration_test.go | 348 ------------------ internal/cmd/jafar/iptables/iptables_linux.go | 119 ------ .../jafar/iptables/iptables_unsupported.go | 45 --- internal/cmd/jafar/main.go | 289 --------------- internal/cmd/jafar/main_test.go | 98 ----- internal/cmd/jafar/resolver/resolver.go | 134 ------- internal/cmd/jafar/resolver/resolver_test.go | 174 --------- internal/cmd/jafar/tlsproxy/tlsproxy.go | 200 ---------- internal/cmd/jafar/tlsproxy/tlsproxy_test.go | 181 --------- internal/cmd/jafar/uncensored/uncensored.go | 72 ---- .../cmd/jafar/uncensored/uncensored_test.go | 63 ---- internal/cmd/miniooni/README.md | 3 +- internal/cmd/tinyjafar/README.md | 113 ++++++ internal/cmd/tinyjafar/main.go | 188 ++++++++++ internal/cmd/tinyjafar/main_test.go | 254 +++++++++++++ internal/engine/.gitignore | 12 - internal/experiment/tor/tor_test.go | 8 +- internal/{cmd/jafar => }/flagx/stringarray.go | 0 .../{cmd/jafar => }/flagx/stringarray_test.go | 6 +- internal/shellx/shellx.go | 6 +- internal/tutorial/README.md | 4 +- .../experiment/torsf/chapter01/README.md | 2 +- .../experiment/torsf/chapter03/README.md | 2 +- .../experiment/torsf/chapter04/README.md | 2 +- internal/tutorial/measurex/README.md | 3 + .../tutorial/measurex/chapter04/README.md | 4 +- internal/tutorial/measurex/chapter04/main.go | 4 +- .../tutorial/netxlite/chapter02/README.md | 8 +- internal/tutorial/netxlite/chapter02/main.go | 8 +- .../tutorial/netxlite/chapter03/README.md | 8 +- internal/tutorial/netxlite/chapter03/main.go | 8 +- script/testjafar.bash | 129 ------- 51 files changed, 601 insertions(+), 3102 deletions(-) delete mode 100644 .github/workflows/jafar.yml delete mode 100644 .github/workflows/qa.yml delete mode 100644 QA/.dockerignore delete mode 100644 QA/.gitignore delete mode 100644 QA/README.md delete mode 100644 QA/common.py delete mode 100755 QA/dockermain.sh delete mode 100755 QA/minioonilike.py delete mode 100755 QA/rundocker.bash delete mode 100755 QA/webconnectivity.py delete mode 100644 internal/cmd/jafar/.gitignore delete mode 100644 internal/cmd/jafar/README.md delete mode 100644 internal/cmd/jafar/badproxy/badproxy.go delete mode 100644 internal/cmd/jafar/badproxy/badproxy_test.go delete mode 100644 internal/cmd/jafar/httpproxy/httpproxy.go delete mode 100644 internal/cmd/jafar/httpproxy/httpproxy_test.go delete mode 100644 internal/cmd/jafar/iptables/iptables.go delete mode 100644 internal/cmd/jafar/iptables/iptables_integration_test.go delete mode 100644 internal/cmd/jafar/iptables/iptables_linux.go delete mode 100644 internal/cmd/jafar/iptables/iptables_unsupported.go delete mode 100644 internal/cmd/jafar/main.go delete mode 100644 internal/cmd/jafar/main_test.go delete mode 100644 internal/cmd/jafar/resolver/resolver.go delete mode 100644 internal/cmd/jafar/resolver/resolver_test.go delete mode 100644 internal/cmd/jafar/tlsproxy/tlsproxy.go delete mode 100644 internal/cmd/jafar/tlsproxy/tlsproxy_test.go delete mode 100644 internal/cmd/jafar/uncensored/uncensored.go delete mode 100644 internal/cmd/jafar/uncensored/uncensored_test.go create mode 100644 internal/cmd/tinyjafar/README.md create mode 100644 internal/cmd/tinyjafar/main.go create mode 100644 internal/cmd/tinyjafar/main_test.go rename internal/{cmd/jafar => }/flagx/stringarray.go (100%) rename internal/{cmd/jafar => }/flagx/stringarray_test.go (88%) delete mode 100755 script/testjafar.bash diff --git a/.github/workflows/jafar.yml b/.github/workflows/jafar.yml deleted file mode 100644 index 546c5adeb9..0000000000 --- a/.github/workflows/jafar.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Checks whether the jafar tool is still WAI. -name: jafar -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - - name: Get GOVERSION content - id: goversion - run: echo "version=$(cat GOVERSION)" >> "$GITHUB_OUTPUT" - - - uses: magnetikonline/action-golang-cache@v4 - with: - go-version: "${{ steps.goversion.outputs.version }}" - cache-key-suffix: "-jafar-${{ steps.goversion.outputs.version }}" - - - run: go build -v ./internal/cmd/jafar - - - run: sudo ./script/testjafar.bash diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml deleted file mode 100644 index 9c98c07bc7..0000000000 --- a/.github/workflows/qa.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Runs quality assurance checks -name: "qa" -on: - push: - branches: - - "release/**" - - "fullbuild" - - "qabuild" - -jobs: - test_webconnectivity: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - run: ./QA/rundocker.bash "webconnectivity" diff --git a/.gitignore b/.gitignore index a011757c85..d632bd8b77 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ /citizenlab-test-lists /gardener /ghpublish.out.txt -/jafar /libooniengine.* /measurement.json /miniooni @@ -25,4 +24,5 @@ /ooniprobe /ooporthelper /probe-cli.cov +/tinyjafar /tmp-* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e2e41acae6..0652f4877a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,10 +88,9 @@ will understand you want to use the default pool) ## Code testing requirements Make sure all tests pass with `go test -race ./...` run from the -top-level directory of this repository. (Integration tests may be -flaky, so there may be some failures here and and there; we know -in particular that `./internal/cmd/jafar` is one of the usual -suspects and that it's not super pleasant to test it under Linux.) +top-level directory of this repository. (Running [netem]( +https://github.com/ooni/netem) based tests may not work as intended +with `-race` with macOS.) ## Writing a new OONI experiment diff --git a/QA/.dockerignore b/QA/.dockerignore deleted file mode 100644 index 72e8ffc0db..0000000000 --- a/QA/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/QA/.gitignore b/QA/.gitignore deleted file mode 100644 index 50dab8e9c0..0000000000 --- a/QA/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/Dockerfile -/GOPATH -/GOCACHE -/__pycache__ diff --git a/QA/README.md b/QA/README.md deleted file mode 100644 index 4600db93da..0000000000 --- a/QA/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Quality assurance scripts - -This directory contains quality assurance scripts that use Jafar to -ensure that OONI implementations behave. These scripts work with miniooni. - -## Run QA using a docker container - -Run test in a suitable Docker container using: - -```bash -./QA/rundocker.bash $nettest -``` - -Note that this will run a `--privileged` docker container. - -## Diagnosing issues - -The Python script that performs the QA runs a specific OONI test under -different failure conditions and stops at the first unexpected value found -in the resulting JSONL report. You can infer what went wrong by reading -the output of the `miniooni` command itself, which should be above the point -where the Python script stopped, as well as by inspecting the JSONL file on -disk. By convention such file is named `$nettest.jsonl` and only contains -the result of the last run of `$nettest`. diff --git a/QA/common.py b/QA/common.py deleted file mode 100644 index 656cb0027a..0000000000 --- a/QA/common.py +++ /dev/null @@ -1,68 +0,0 @@ -""" ./QA/common.py - common code for QA """ - -import contextlib -import json -import os -import shutil -import socket -import subprocess - - -def execute(args): - """ Execute a specified command """ - subprocess.run(args) - - -def execute_jafar_and_miniooni(ooni_exe, outfile, experiment, tag, args): - """ Executes jafar and miniooni. Returns the test keys. """ - tmpoutfile = "/tmp/{}".format(outfile) - with contextlib.suppress(FileNotFoundError): - os.remove(tmpoutfile) # just in case - execute( - [ - "./jafar", - "-main-command", - "./QA/minioonilike.py {} -n -o '{}' --home /tmp {}".format( - ooni_exe, tmpoutfile, experiment - ), - "-main-user", - "nobody", # should be present on Unix - "-tag", - tag, - ] - + args - ) - shutil.copy(tmpoutfile, outfile) - result = read_result(outfile) - assert isinstance(result, dict) - assert isinstance(result["test_keys"], dict) - return result["test_keys"] - - -def read_result(outfile): - """ Reads the result of an experiment """ - return json.load(open(outfile, "rb")) - - -def test_keys(result): - """ Returns just the test keys of a specific result """ - return result["test_keys"] - - -def check_maybe_binary_value(value): - """ Make sure a maybe binary value is correct """ - assert isinstance(value, str) or ( - isinstance(value, dict) - and value["format"] == "base64" - and isinstance(value["data"], str) - ) - - -def with_free_port(func): - """ This function executes |func| passing it a port number on localhost - which is bound but not listening for new connections """ - # See - with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - sock.bind(("127.0.0.1", 0)) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - func(sock.getsockname()[1]) diff --git a/QA/dockermain.sh b/QA/dockermain.sh deleted file mode 100755 index fab293da29..0000000000 --- a/QA/dockermain.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -set -euxo pipefail - -# required because the container is running as root -git config --global --add safe.directory /jafar - -# TODO(bassosimone): investigate why using CGO_ENABLED=1 is such -# that all DNS lookups return `dns_nxdomain_error` -export CGO_ENABLED=0 - -# TODO(bassosimone): because this script runs as root, it's not -# possible to save the caching directories in github actions but -# doing that would making re-executing these scripts faster. -export GOPATH=/jafar/QA/GOPATH -export GOCACHE=/jafar/QA/GOCACHE - -go build -v ./internal/cmd/miniooni - -go build -v ./internal/cmd/jafar - -sudo ./QA/$1.py ./miniooni diff --git a/QA/minioonilike.py b/QA/minioonilike.py deleted file mode 100755 index a4b9dda3c1..0000000000 --- a/QA/minioonilike.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 - -""" This script takes in input the name of the tool to run followed by - arguments and followed by the nettest name. The format recognized is - the same of miniooni. Depending on the tool that we want to run, we - reorder arguments so that they make sense for the tool. - - This is necessary because, albeit miniooni, MK, and OONI v2.x have - more or less the same arguments, there are some differences. We could - modify other tools to match miniooni, but this seems useless. """ - -import argparse -import os -import shlex -import sys - -sys.path.insert(0, ".") -import common - - -def file_must_exist(pathname): - """ Throws an exception if the given file does not actually exist. """ - if not os.path.isfile(pathname): - raise RuntimeError("missing {}: please run miniooni first".format(pathname)) - return pathname - - -def main(): - apa = argparse.ArgumentParser() - apa.add_argument("command", nargs=1, help="command to execute") - - # subset of arguments accepted by miniooni - apa.add_argument( - "-n", "--no-collector", action="count", help="don't submit measurement" - ) - apa.add_argument("-o", "--reportfile", help="specify report file to use") - apa.add_argument("-i", "--input", help="input for nettests taking an input") - apa.add_argument("--home", help="override home directory") - apa.add_argument("nettest", nargs=1, help="nettest to run") - out = apa.parse_args() - command, nettest = out.command[0], out.nettest[0] - - if "miniooni" not in command and "measurement_kit" not in command: - raise RuntimeError("unrecognized tool") - - args = [] - args.append(command) - if "miniooni" in command: - args.extend(["--yes"]) # make sure we have informed consent - if "measurement_kit" in command: - args.extend( - [ - "--ca-bundle-path", - file_must_exist("{}/.miniooni/assets/ca-bundle.pem".format(out.home)), - ] - ) - args.extend( - [ - "--geoip-country-path", - file_must_exist("{}/.miniooni/assets/country.mmdb".format(out.home)), - ] - ) - args.extend( - [ - "--geoip-asn-path", - file_must_exist("{}/.miniooni/assets/asn.mmdb".format(out.home)), - ] - ) - if out.home and "miniooni" in command: - args.extend(["--home", out.home]) # home applies to miniooni only - if out.input: - if "miniooni" in command: - args.extend(["-i", out.input]) # input is -i for miniooni - if out.no_collector: - args.append("-n") - if out.reportfile: - args.extend(["-o", out.reportfile]) - args.append(nettest) - if out.input and "measurement_kit" in command: - if nettest == "web_connectivity": - args.extend(["-u", out.input]) # MK's Web Connectivity uses -u for input - - sys.stderr.write("minioonilike.py: {}\n".format(shlex.join(args))) - common.execute(args) - - -if __name__ == "__main__": - main() diff --git a/QA/rundocker.bash b/QA/rundocker.bash deleted file mode 100755 index 77ec540b55..0000000000 --- a/QA/rundocker.bash +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -DOCKER=${DOCKER:-docker} - -GOVERSION=$(cat GOVERSION) - -cat > QA/Dockerfile << EOF -FROM golang:$GOVERSION-alpine -RUN apk add gcc go git musl-dev iptables tmux bind-tools curl sudo python3 -EOF - -$DOCKER build -t jafar-qa ./QA/ - -$DOCKER run --privileged -v$(pwd):/jafar -w/jafar jafar-qa ./QA/dockermain.sh "$@" diff --git a/QA/webconnectivity.py b/QA/webconnectivity.py deleted file mode 100755 index 3c8555cc07..0000000000 --- a/QA/webconnectivity.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/webconnectivity.py - main QA script for webconnectivity - - This script performs a bunch of webconnectivity tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import socket -import sys -import time - -sys.path.insert(0, ".") -import common - - -def execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, experiment_args, tag, args -): - """Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid.""" - tk = common.execute_jafar_and_miniooni( - ooni_exe, outfile, experiment_args, tag, args - ) - print("dns_experiment_failure", tk["dns_experiment_failure"], file=sys.stderr) - print("dns_consistency", tk["dns_consistency"], file=sys.stderr) - print("control_failure", tk["control_failure"], file=sys.stderr) - print("http_experiment_failure", tk["http_experiment_failure"], file=sys.stderr) - print("tk_body_length_match", tk["body_length_match"], file=sys.stderr) - print("body_proportion", tk["body_proportion"], file=sys.stderr) - print("status_code_match", tk["status_code_match"], file=sys.stderr) - print("headers_match", tk["headers_match"], file=sys.stderr) - print("title_match", tk["title_match"], file=sys.stderr) - print("blocking", tk["blocking"], file=sys.stderr) - print("accessible", tk["accessible"], file=sys.stderr) - print("x_status", tk["x_status"], file=sys.stderr) - return tk - - -def assert_status_flags_are(ooni_exe, tk, desired): - """Checks whether the status flags are what we expect them to - be when we're running miniooni. This check only makes sense - with miniooni b/c status flags are a miniooni extension.""" - if "miniooni" not in ooni_exe: - return - assert tk["x_status"] == desired - - -def webconnectivity_https_self_signed(ooni_exe, outfile): - """Test case where the certificate is self signed""" - args = [] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, - outfile, - "-i https://self-signed.badssl.com/ web_connectivity", - "webconnectivity_https_self_signed", - args, - ) - assert tk["dns_experiment_failure"] == None - assert tk["dns_consistency"] == "consistent" - assert tk["control_failure"] == None - if "miniooni" in ooni_exe: - assert tk["http_experiment_failure"] == "ssl_unknown_authority" - else: - assert "certificate verify failed" in tk["http_experiment_failure"] - assert tk["body_length_match"] == None - assert tk["body_proportion"] == 0 - assert tk["status_code_match"] == None - assert tk["headers_match"] == None - assert tk["title_match"] == None - # The following strikes me as a measurement_kit bug. We are saying - # that all is good with a domain where actually we don't know why the - # control is failed and that is clearly not accessible according to - # our measurement of the domain (self signed certificate). - # - # See . - if "miniooni" in ooni_exe: - assert tk["blocking"] == None - assert tk["accessible"] == None - else: - assert tk["blocking"] == False - assert tk["accessible"] == True - assert_status_flags_are(ooni_exe, tk, 16) - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "webconnectivity.jsonl" - ooni_exe = sys.argv[1] - tests = [ - webconnectivity_https_self_signed, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/internal/cmd/jafar/.gitignore b/internal/cmd/jafar/.gitignore deleted file mode 100644 index 8e4647b852..0000000000 --- a/internal/cmd/jafar/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/badproxy.pem diff --git a/internal/cmd/jafar/README.md b/internal/cmd/jafar/README.md deleted file mode 100644 index 01081bdfc4..0000000000 --- a/internal/cmd/jafar/README.md +++ /dev/null @@ -1,257 +0,0 @@ -# Jafar - -> We stepped up the game of simulating censorship upgrading from the -> evil genius to the evil grand vizier. - -Jafar is a censorship simulation tool used for testing OONI. It builds on -any system but it really only works on Linux. - -## Building - -We use Go >= 1.20. Jafar also needs the C library headers, -iptables installed, and root permissions. - -With Linux Alpine edge, you can compile Jafar with: - -```bash -apk add go git musl-dev iptables -go build -v . -``` - -Otherwise, using Docker: - -```bash -docker build -t jafar-runner . -docker run -it --privileged -v`pwd`:/jafar -w/jafar jafar-runner -go build -v . -``` - -## Usage - -You need to run Jafar as root. You can get a complete list -of all flags using `./jafar -help`. Jafar is composed of modules. Each -module is controllable via flags. We describe modules below. - -### main - -The main module starts all the other modules. If you don't provide the -`-main-command ` flag, the code will run until interrupted. If -instead you use the `-main-command` flag, you can specify a command to -run inside the censored environment. In such case, the main module -will exit when the specified command terminates. Note that the main -module will propagate the child exit code, if the child fails. - -The command can also include arguments. Make sure you quote the arguments -such that your shell passes the whole string to the specified option, as -in `-main-command 'ls -lha'`. This will execute the `ls -lha` command line -inside the censored Jafar context. You can also combine that with quoting -and variables interpolation, e.g., `-main-command "echo '$USER is the -walrus'"`. The `$USER` variable will be expanded by your shell. Assuming -your user name is `paul`, then Jafar will lex the main command as `echo -"paul is the walrus"` and will execute it. - -Use the `-main-user ` flag to select the user to use for -running child commands. By default, we use the `nobody` user for this -purpose. We implement this feature using `sudo`, therefore you need -to make sure that `sudo` is installed. - -### iptables - -The iptables module is only available on Linux. It exports these flags: - -```bash - -iptables-drop-ip value - Drop traffic to the specified IP address - -iptables-drop-keyword-hex value - Drop traffic containing the specified hex keyword - -iptables-drop-keyword value - Drop traffic containing the specified keyword - -iptables-hijack-dns-to string - Hijack all DNS UDP traffic to the specified endpoint - -iptables-hijack-https-to string - Hijack all HTTPS traffic to the specified endpoint - -iptables-hijack-http-to string - Hijack all HTTP traffic to the specified endpoint - -iptables-reset-ip value - Reset TCP/IP traffic to the specified IP address - -iptables-reset-keyword-hex value - Reset TCP/IP traffic containing the specified hex keyword - -iptables-reset-keyword value - Reset TCP/IP traffic containing the specified keyword -``` - -The difference between `drop` and `reset` is that in the former case -a packet is dropped, in the latter case a RST is sent. - -The difference between `ip` and `keyword` flags is that the former -match an outgoing IP, the latter uses DPI. - -The `drop` and `reset` rules allow you to simulate, respectively, when -operations timeout and when a connection cannot be established (with -`reset` and `ip`) or is reset after a keyword is seen (with `keyword`). - -Hijacking DNS traffic is useful, for example, to redirect all DNS UDP -traffic from the box to the `dns-proxy` module. - -Hijacking HTTP and HTTPS traffic actually hijacks based on ports rather -than on DPI. As a known bug, when hijacking HTTP or HTTPS traffic, we -do not hijack traffic owned by root. This is because Jafar runs as root -and therefore its traffic must not match the hijack rule. - -When matching keywords, the simplest option is to use ASCII strings as -in `-iptables-drop-keyword ooni`. However, you can also specify a sequence -of hex bytes, as in `-iptables-drop-keyword-hex |6f 6f 6e 69|`. - -Note that with `-iptables-drop-keyword`, DNS queries containing such -keyword will fail returning `EPERM`. For a more realistic approach to -dropping specific DNS packets, combine DNS traffic hijacking with -`-dns-proxy-ignore`, to "drop" packets at the DNS proxy. - -### dns-proxy (aka resolver) - -The DNS proxy or resolver allows to manipulate DNS. Unless you use DNS -hijacking, you will need to configure your application explicitly to use -the proxy with application specific command line flags. - -```bash - -dns-proxy-address string - Address where the DNS proxy should listen (default "127.0.0.1:53") - -dns-proxy-block value - Register keyword triggering NXDOMAIN censorship - -dns-proxy-hijack value - Register keyword triggering redirection to 127.0.0.1 - -dns-proxy-ignore value - Register keyword causing the proxy to ignore the query -``` - -The `-dns-proxy-address` flag controls the endpoint where the proxy is -listening. - -The `-dns-proxy-block` tells the resolver that every incoming request whose -query contains the specifed string shall receive an `NXDOMAIN` reply. - -The `-dns-proxy-hijack` is similar but instead lies and returns to the -client that the requested domain is at `127.0.0.1`. This is an opportunity -to redirect traffic to the HTTP and TLS proxies. - -The `-dns-proxy-ignore` is similar but instead just ignores the query. - -### http-proxy - -The HTTP proxy is an HTTP proxy that may refuse to forward some -specific requests. It's controlled by these flags: - -```bash - -http-proxy-address string - Address where the HTTP proxy should listen (default "127.0.0.1:80") - -http-proxy-block value - Register keyword triggering HTTP 451 censorship -``` - -The `-http-proxy-address` flag has the same semantics it has for the DNS -proxy. - -The `-http-proxy-block` flag tells the proxy that it should return a `451` -response for every request whose `Host` contains the specified string. - -### tls-proxy - -TLS proxy is a TCP proxy that routes traffic to specific servers depending -on their SNI value. It is controlled by the following flags: - -```bash - -tls-proxy-address string - Address where the TCP+TLS proxy should listen (default "127.0.0.1:443") - -tls-proxy-block value - Register SNI header keyword triggering TLS censorship - -tls-proxy-outbound-port - Define the outbound port requests are proxied to (default "443" for HTTPS) -``` - -The `-tls-proxy-address` flags has the same semantics it has for the DNS -proxy. - -The `-tls-proxy-block` specifies which string or strings should cause the -proxy to return an internal-erorr alert when the incoming ClientHello's SNI -contains one of the strings provided with this option. - -### bad-proxy - -```bash - -bad-proxy-address string - Address where to listen for TCP connections (default "127.0.0.1:7117") - -bad-proxy-address-tls string - Address where to listen for TLS connections (default "127.0.0.1:4114") - -bad-proxy-tls-output-ca string - File where to write the CA used by the bad proxy (default "badproxy.pem") -``` - -The bad proxy is a proxy that reads some bytes from any incoming connection -and then closes the connection without replying anything. This simulates a -proxy that is not working properly, hence the name of the module. - -When connecting using TLS, the above behaviour happens after the handshake. - -We write the CA on the file specified using `-bad-proxy-tls-output-ca` such that -tools like curl(1) can use such CA to avoid TLS handshake errors. The code will -generate on the fly a certificate for the provided SNI. Not providing any SNI in -the client Hello message will cause the TLS handshake to fail. - -### uncensored - -```bash - -uncensored-resolver-doh string - URL of an hopefully uncensored DoH resolver (default "https://1.1.1.1/dns-query") -``` - -The HTTP, DNS, and TLS proxies need to resolve domain names. If you setup DNS -censorship, they may be affected as well. To avoid this issue, we use a different -resolver for them, which by default is the one shown above. You can change such -default by using the `-uncensored-resolver-doh` command line flag. The input -URL is an HTTPS URL pointing to a DoH server. Here are some examples: - -* `https://dns.google/dns-query` -* `https://dns.quad9.net/dns-query` - -So, for example, if you are using Jafar to censor `1.1.1.1:443`, then you -most likely want to use `-uncensored-resolver-doh`. - -## Examples - -Block `play.google.com` with RST injection, force DNS traffic to use the our -DNS proxy, and force it to censor `play.google.com` with `NXDOMAIN`. - -```bash -# ./jafar -iptables-reset-keyword play.google.com \ - -iptables-hijack-dns-to 127.0.0.1:5353 \ - -dns-proxy-address 127.0.0.1:5353 \ - -dns-proxy-block play.google.com -``` - -Force all traffic through the HTTP and TLS proxy and use them to censor -`play.google.com` using HTTP 451 and responding with TLS alerts: - -```bash -# ./jafar -iptables-hijack-dns-to 127.0.0.1:5353 \ - -dns-proxy-address 127.0.0.1:5353 \ - -dns-proxy-hijack play.google.com \ - -http-proxy-block play.google.com \ - -tls-proxy-block play.google.com -``` - -Run `ping` in a censored environment: - -```bash -# ./jafar -iptables-drop-ip 8.8.8.8 -main-command 'ping -c3 8.8.8.8' -``` - -Run `curl` in a censored environment where it cannot connect to -`play.google.com` using `https`: - -```bash -# ./jafar -iptables-hijack-https-to 127.0.0.1:443 \ - -tls-proxy-block play.google.com \ - -main-command 'curl -Lv http://play.google.com' -``` - -For more usage examples, see `../../script/testjafar.bash`. diff --git a/internal/cmd/jafar/badproxy/badproxy.go b/internal/cmd/jafar/badproxy/badproxy.go deleted file mode 100644 index 3690bb63e8..0000000000 --- a/internal/cmd/jafar/badproxy/badproxy.go +++ /dev/null @@ -1,114 +0,0 @@ -// Package badproxy implements misbehaving proxies. We have a single -// CensoringProxy that exports two misbehaving endpoints. Each endpoint -// implements a different proxy-censorsing technique. The first one -// reads some bytes from the connection then closes the connection. The -// other instead replies with a self signed x509 certificate. -package badproxy - -import ( - "context" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "io" - "net" - "strings" - "time" - - "github.com/google/martian/v3/mitm" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// CensoringProxy is a proxy that does not behave correctly. -type CensoringProxy struct { - mitmNewAuthority func( - name string, organization string, - validity time.Duration, - ) (*x509.Certificate, *rsa.PrivateKey, error) - - mitmNewConfig func( - ca *x509.Certificate, privateKey interface{}, - ) (*mitm.Config, error) - - tlsListen func( - network string, laddr string, config *tls.Config, - ) (net.Listener, error) -} - -// NewCensoringProxy creates a new instance of a misbehaving proxy. -func NewCensoringProxy() *CensoringProxy { - return &CensoringProxy{ - mitmNewAuthority: mitm.NewAuthority, - mitmNewConfig: mitm.NewConfig, - tlsListen: tls.Listen, - } -} - -func (p *CensoringProxy) serve(conn net.Conn) { - deadline := time.Now().Add(250 * time.Millisecond) - conn.SetDeadline(deadline) - // To simulate the case where the proxy isn't willing to forward our - // traffic, we close the connection (1) right after the handshake for - // TLS connections and (2) reasonably after we've received the HTTP - // request for cleartext connections. This may break in several cases - // but is good enough approximation of these bad proxies for now. - if tlsconn, ok := conn.(*tls.Conn); ok { - tlsconn.Handshake() - } else { - const maxread = 1 << 17 - reader := io.LimitReader(conn, maxread) - netxlite.ReadAllContext(context.Background(), reader) - } - conn.Close() -} - -func (p *CensoringProxy) run(listener net.Listener) { - for { - conn, err := listener.Accept() - if err != nil && strings.Contains( - err.Error(), "use of closed network connection") { - return - } - if err == nil { - // It's difficult to make accept fail, so restructure - // the code such that we enter into the happy path - go p.serve(conn) - } - } -} - -// Start starts the misbehaving proxy for TCP. This endpoint will read some -// bytes from the request and then close the connection. This behaviour is -// implemented by a bunch of censoring proxy around the world. Usually such -// proxies only close the connection with offending SNIs/Host headers. -func (p *CensoringProxy) Start(address string) (net.Listener, error) { - listener, err := net.Listen("tcp", address) - if err != nil { - return nil, err - } - go p.run(listener) - return listener, nil -} - -// StartTLS starts the misbehaving proxy for TLS. This endpoint will return -// to the client a self signed certificate. Thus, it models the case where a -// MITM forces users to accept a rogue certificate. After sending such a -// certificate, this proxy will close the TCP connection. -func (p *CensoringProxy) StartTLS(address string) (net.Listener, *x509.Certificate, error) { - cert, privkey, err := p.mitmNewAuthority( - "jafar", "OONI", 24*time.Hour, - ) - if err != nil { - return nil, nil, err - } - config, err := p.mitmNewConfig(cert, privkey) - if err != nil { - return nil, nil, err - } - listener, err := p.tlsListen("tcp", address, config.TLS()) - if err != nil { - return nil, nil, err - } - go p.run(listener) - return listener, cert, nil -} diff --git a/internal/cmd/jafar/badproxy/badproxy_test.go b/internal/cmd/jafar/badproxy/badproxy_test.go deleted file mode 100644 index 2b0769ba64..0000000000 --- a/internal/cmd/jafar/badproxy/badproxy_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package badproxy - -import ( - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "errors" - "net" - "testing" - "time" - - "github.com/google/martian/v3/mitm" -) - -func TestCleartext(t *testing.T) { - listener := newproxy(t) - checkdial(t, listener.Addr().String(), nil, net.Dial) - killproxy(t, listener) -} - -func TestTLS(t *testing.T) { - listener := newproxytls(t) - checkdial(t, listener.Addr().String(), nil, - func(network, address string) (net.Conn, error) { - conn, err := tls.Dial(network, address, &tls.Config{ - InsecureSkipVerify: true, - ServerName: "antani.local", - }) - if err != nil { - return nil, err - } - if err = conn.Handshake(); err != nil { - conn.Close() - return nil, err - } - return conn, nil - }) - killproxy(t, listener) -} - -func TestListenError(t *testing.T) { - proxy := NewCensoringProxy() - listener, err := proxy.Start("8.8.8.8:80") - if err == nil { - t.Fatal("expected an error here") - } - if listener != nil { - t.Fatal("expected nil listener here") - } -} - -func TestStartTLS(t *testing.T) { - expected := errors.New("mocked error") - - t.Run("when we cannot create a new authority", func(t *testing.T) { - proxy := NewCensoringProxy() - proxy.mitmNewAuthority = func( - name string, organization string, - validity time.Duration, - ) (*x509.Certificate, *rsa.PrivateKey, error) { - return nil, nil, expected - } - cert, privkey, err := proxy.StartTLS("127.0.0.1:0") - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } - if cert != nil { - t.Fatal("expected nil cert") - } - if privkey != nil { - t.Fatal("expected nil privkey") - } - }) - - t.Run("when we cannot create a new config", func(t *testing.T) { - proxy := NewCensoringProxy() - proxy.mitmNewConfig = func( - ca *x509.Certificate, privateKey interface{}, - ) (*mitm.Config, error) { - return nil, expected - } - cert, privkey, err := proxy.StartTLS("127.0.0.1:0") - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } - if cert != nil { - t.Fatal("expected nil cert") - } - if privkey != nil { - t.Fatal("expected nil privkey") - } - }) - - t.Run("when we cannot listen", func(t *testing.T) { - proxy := NewCensoringProxy() - proxy.tlsListen = func( - network string, laddr string, config *tls.Config, - ) (net.Listener, error) { - return nil, expected - } - cert, privkey, err := proxy.StartTLS("127.0.0.1:0") - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } - if cert != nil { - t.Fatal("expected nil cert") - } - if privkey != nil { - t.Fatal("expected nil privkey") - } - }) -} - -func newproxy(t *testing.T) net.Listener { - proxy := NewCensoringProxy() - listener, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return listener -} - -func newproxytls(t *testing.T) net.Listener { - proxy := NewCensoringProxy() - listener, _, err := proxy.StartTLS("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return listener -} - -func killproxy(t *testing.T, listener net.Listener) { - err := listener.Close() - if err != nil { - t.Fatal(err) - } -} - -func checkdial( - t *testing.T, proxyAddr string, expectErr error, - dial func(network, address string) (net.Conn, error), -) { - conn, err := dial("tcp", proxyAddr) - if err != expectErr { - t.Fatal("not the result we expected") - } - if conn == nil && expectErr == nil { - t.Fatal("expected actionable conn") - } - if conn != nil && expectErr != nil { - t.Fatal("expected nil conn") - } - if conn != nil { - conn.Write([]byte("123454321")) - conn.Close() - } -} diff --git a/internal/cmd/jafar/httpproxy/httpproxy.go b/internal/cmd/jafar/httpproxy/httpproxy.go deleted file mode 100644 index eac502510a..0000000000 --- a/internal/cmd/jafar/httpproxy/httpproxy.go +++ /dev/null @@ -1,78 +0,0 @@ -// Package httpproxy contains a censoring HTTP proxy. This proxy will -// vet all the traffic and reply with 451 responses for a configurable -// set of offending Host headers in incoming requests. -package httpproxy - -import ( - "net" - "net/http" - "net/http/httputil" - "net/url" - "strings" -) - -const product = "jafar/0.1.0" - -// CensoringProxy is a censoring HTTP proxy -type CensoringProxy struct { - keywords []string - transport http.RoundTripper -} - -// NewCensoringProxy creates a new CensoringProxy instance using -// the specified list of keywords to censor. keywords is the list -// of keywords that trigger censorship if any of them appears in -// the Host header of a request. dnsNetwork and dnsAddress are -// settings to configure the upstream, non censored DNS. -func NewCensoringProxy( - keywords []string, uncensored http.RoundTripper, -) *CensoringProxy { - return &CensoringProxy{keywords: keywords, transport: uncensored} -} - -var blockpage = []byte(` - 451 Unavailable For Legal Reasons - -

451 Unavailable For Legal Reasons

-

This content is not available in your jurisdiction.

- -`) - -// ServeHTTP serves HTTP requests -func (p *CensoringProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Implementation note: use Via header to detect in a loose way - // requests originated by us and directed to us - if r.Header.Get("Via") != "" || r.Host == "" { - w.WriteHeader(http.StatusBadRequest) - return - } - for _, pattern := range p.keywords { - if strings.Contains(r.Host, pattern) { - w.WriteHeader(http.StatusUnavailableForLegalReasons) - w.Write(blockpage) - return - } - } - r.Header.Add("Via", product) // see above - proxy := httputil.NewSingleHostReverseProxy(&url.URL{ - Host: r.Host, - Scheme: "http", - }) - proxy.ModifyResponse = func(resp *http.Response) error { - resp.Header.Add("Via", product) // see above - return nil - } - proxy.Transport = p.transport - proxy.ServeHTTP(w, r) -} - -// Start starts the censoring proxy. -func (p *CensoringProxy) Start(address string) (*http.Server, net.Addr, error) { - server := &http.Server{Handler: p} - listener, err := net.Listen("tcp", address) - if err != nil { - return nil, nil, err - } - go server.Serve(listener) - return server, listener.Addr(), nil -} diff --git a/internal/cmd/jafar/httpproxy/httpproxy_test.go b/internal/cmd/jafar/httpproxy/httpproxy_test.go deleted file mode 100644 index 399f78b43b..0000000000 --- a/internal/cmd/jafar/httpproxy/httpproxy_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package httpproxy - -import ( - "bytes" - "context" - "net" - "net/http" - "testing" - - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestPass(t *testing.T) { - server, addr := newproxy(t, "ooni.io") - // We're filtering ooni.io, so we expect example.com to pass - // through the proxy with 200 and we also expect to see the - // Via header in the responses we receive, of course. - checkrequest(t, addr.String(), "example.com", 200, true) - killproxy(t, server) -} - -func TestBlock(t *testing.T) { - server, addr := newproxy(t, "ooni.io") - // Here we're filtering any domain containing ooni.io, so we - // expect the proxy to send 451 without actually proxing, thus - // there should not be any Via header in the output. - checkrequest(t, addr.String(), "api.ooni.io", 451, false) - killproxy(t, server) -} - -func TestLoop(t *testing.T) { - server, addr := newproxy(t, "ooni.io") - // Here we're forcing the proxy to connect to itself. It does - // does that and recognizes itself because of the Via header - // being set in the request generated by the connection to itself, - // which should cause a 400. The response should have the Via - // header set because the 400 is received by the connection that - // this code has made to the proxy. - checkrequest(t, addr.String(), addr.String(), 400, true) - killproxy(t, server) -} - -func TestListenError(t *testing.T) { - proxy := NewCensoringProxy([]string{""}, uncensored.NewClient("https://1.1.1.1/dns-query")) - server, addr, err := proxy.Start("8.8.8.8:80") - if err == nil { - t.Fatal("expected an error here") - } - if server != nil { - t.Fatal("expected nil server here") - } - if addr != nil { - t.Fatal("expected nil addr here") - } -} - -func newproxy(t *testing.T, blocked string) (*http.Server, net.Addr) { - proxy := NewCensoringProxy([]string{blocked}, uncensored.NewClient("https://1.1.1.1/dns-query")) - server, addr, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return server, addr -} - -func killproxy(t *testing.T, server *http.Server) { - err := server.Shutdown(context.Background()) - if err != nil { - t.Fatal(err) - } -} - -func checkrequest( - t *testing.T, proxyAddr, host string, - expectStatus int, expectVia bool, -) { - req, err := http.NewRequest("GET", "http://"+proxyAddr, nil) - if err != nil { - t.Fatal(err) - } - req.Host = host - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != expectStatus { - t.Fatal("unexpected value of status code") - } - t.Log(resp) - values := resp.Header["Via"] - var foundProduct bool - for _, value := range values { - if value == product { - foundProduct = true - } - } - if foundProduct && !expectVia { - t.Fatal("unexpectedly found Via header") - } - if !foundProduct && expectVia { - t.Fatal("Via header not found") - } - proxiedData, err := netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - if expectStatus == 200 { - checkbody(t, proxiedData, host) - } -} - -func checkbody(t *testing.T, proxiedData []byte, host string) { - resp, err := http.Get("http://" + host) - if err != nil { - t.Fatal(err) - } - if resp.StatusCode != 200 { - t.Fatal("unexpected status code") - } - defer resp.Body.Close() - data, err := netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - if bytes.Equal(data, proxiedData) == false { - t.Fatal("body mismatch") - } -} diff --git a/internal/cmd/jafar/iptables/iptables.go b/internal/cmd/jafar/iptables/iptables.go deleted file mode 100644 index 580e8bf1df..0000000000 --- a/internal/cmd/jafar/iptables/iptables.go +++ /dev/null @@ -1,98 +0,0 @@ -// Package iptables contains code for managing firewall rules. This package -// really only works reliably on Linux. In all other systems the functionality -// in here is just a set of stubs returning errors. -package iptables - -import ( - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -type shell interface { - createChains() error - dropIfDestinationEquals(ip string) error - rstIfDestinationEqualsAndIsTCP(ip string) error - dropIfContainsKeywordHex(keyword string) error - dropIfContainsKeyword(keyword string) error - rstIfContainsKeywordHexAndIsTCP(keyword string) error - rstIfContainsKeywordAndIsTCP(keyword string) error - hijackDNS(address string) error - hijackHTTPS(address string) error - hijackHTTP(address string) error - waive() error -} - -// CensoringPolicy implements a censoring policy. -type CensoringPolicy struct { - DropIPs []string // drop IP traffic to these IPs - DropKeywordsHex []string // drop IP packets with these hex keywords - DropKeywords []string // drop IP packets with these keywords - HijackDNSAddress string // where to hijack DNS to - HijackHTTPSAddress string // where to hijack HTTPS to - HijackHTTPAddress string // where to hijack HTTP to - ResetIPs []string // RST TCP/IP traffic to these IPs - ResetKeywordsHex []string // RST TCP/IP flows with these hex keywords - ResetKeywords []string // RST TCP/IP flows with these keywords - sh shell -} - -// NewCensoringPolicy returns a new censoring policy. -func NewCensoringPolicy() *CensoringPolicy { - return &CensoringPolicy{ - sh: newShell(), - } -} - -// Apply applies the censorship policy -func (c *CensoringPolicy) Apply() (err error) { - defer func() { - if recover() != nil { - // JUST KNOW WE'VE BEEN HERE - } - }() - err = c.sh.createChains() - runtimex.PanicOnError(err, "c.sh.createChains failed") - // Implementation note: we want the RST rules to be first such - // that we end up enforcing them before the drop rules. - for _, keyword := range c.ResetKeywordsHex { - err = c.sh.rstIfContainsKeywordHexAndIsTCP(keyword) - runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordHexAndIsTCP failed") - } - for _, keyword := range c.ResetKeywords { - err = c.sh.rstIfContainsKeywordAndIsTCP(keyword) - runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordAndIsTCP failed") - } - for _, ip := range c.ResetIPs { - err = c.sh.rstIfDestinationEqualsAndIsTCP(ip) - runtimex.PanicOnError(err, "c.sh.rstIfDestinationEqualsAndIsTCP failed") - } - for _, keyword := range c.DropKeywordsHex { - err = c.sh.dropIfContainsKeywordHex(keyword) - runtimex.PanicOnError(err, "c.sh.dropIfContainsKeywordHex failed") - } - for _, keyword := range c.DropKeywords { - err = c.sh.dropIfContainsKeyword(keyword) - runtimex.PanicOnError(err, "c.sh.dropIfContainsKeyword failed") - } - for _, ip := range c.DropIPs { - err = c.sh.dropIfDestinationEquals(ip) - runtimex.PanicOnError(err, "c.sh.dropIfDestinationEquals failed") - } - if c.HijackDNSAddress != "" { - err = c.sh.hijackDNS(c.HijackDNSAddress) - runtimex.PanicOnError(err, "c.sh.hijackDNS failed") - } - if c.HijackHTTPSAddress != "" { - err = c.sh.hijackHTTPS(c.HijackHTTPSAddress) - runtimex.PanicOnError(err, "c.sh.hijackHTTPS failed") - } - if c.HijackHTTPAddress != "" { - err = c.sh.hijackHTTP(c.HijackHTTPAddress) - runtimex.PanicOnError(err, "c.sh.hijackHTTP failed") - } - return -} - -// Waive removes any censorship policy -func (c *CensoringPolicy) Waive() error { - return c.sh.waive() -} diff --git a/internal/cmd/jafar/iptables/iptables_integration_test.go b/internal/cmd/jafar/iptables/iptables_integration_test.go deleted file mode 100644 index 50f17228d9..0000000000 --- a/internal/cmd/jafar/iptables/iptables_integration_test.go +++ /dev/null @@ -1,348 +0,0 @@ -package iptables - -import ( - "context" - "errors" - "net" - "net/http" - "net/http/httptest" - "net/url" - "runtime" - "strings" - "testing" - "time" - - "golang.org/x/sys/execabs" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/resolver" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" - "github.com/ooni/probe-cli/v3/internal/shellx" -) - -func init() { - log.SetLevel(log.ErrorLevel) -} - -func newCensoringPolicy() *CensoringPolicy { - policy := NewCensoringPolicy() - policy.Waive() // start over to allow for repeated tests on failure - return policy -} - -func TestCannotApplyPolicy(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.DropIPs = []string{"antani"} - if err := policy.Apply(); err == nil { - t.Fatal("expected an error here") - } -} - -func TestCreateChainsError(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - // you should not be able to apply the policy when there is - // already a policy, you need to waive it first - if err := policy.Apply(); err == nil { - t.Fatal("expected an error here") - } -} - -func TestDropIP(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.DropIPs = []string{"1.1.1.1"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", "1.1.1.1:853") - if err == nil { - t.Fatalf("expected an error here") - } - if err.Error() != "dial tcp 1.1.1.1:853: i/o timeout" { - t.Fatal("unexpected error occurred") - } - if conn != nil { - t.Fatal("expected nil connection here") - } -} - -func TestDropKeyword(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.DropKeywords = []string{"ooni.io"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - req, err := http.NewRequest("GET", "http://www.ooni.io", nil) - if err != nil { - t.Fatal(err) - } - resp, err := http.DefaultClient.Do(req.WithContext(ctx)) - if err == nil { - t.Fatal("expected an error here") - } - if !strings.HasSuffix(err.Error(), "context deadline exceeded") { - t.Fatal("unexpected error occurred") - } - if resp != nil { - t.Fatal("expected nil response here") - } -} - -func TestDropKeywordHex(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.DropKeywordsHex = []string{"|6f 6f 6e 69|"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - reso := &net.Resolver{ - PreferGo: true, - } - addrs, err := reso.LookupHost(ctx, "www.ooni.io") - if err == nil { - t.Fatal("expected an error here") - } - // the error we see with GitHub Actions is different from the error - // we see when testing locally on Fedora - if !strings.HasSuffix(err.Error(), "operation not permitted") && - !strings.HasSuffix(err.Error(), "Temporary failure in name resolution") && - !strings.HasSuffix(err.Error(), "no such host") { - t.Fatalf("unexpected error occurred: %+v", err) - } - if addrs != nil { - t.Fatal("expected nil response here") - } -} - -func TestResetIP(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.ResetIPs = []string{"1.1.1.1"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - conn, err := (&net.Dialer{}).Dial("tcp", "1.1.1.1:853") - if err == nil { - t.Fatalf("expected an error here") - } - if err.Error() != "dial tcp 1.1.1.1:853: connect: connection refused" { - t.Fatal("unexpected error occurred") - } - if conn != nil { - t.Fatal("expected nil connection here") - } -} - -func TestResetKeyword(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.ResetKeywords = []string{"ooni.io"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - resp, err := http.Get("http://www.ooni.io") - if err == nil { - t.Fatal("expected an error here") - } - if strings.Contains(err.Error(), "read: connection reset by peer") == false { - t.Fatal("unexpected error occurred") - } - if resp != nil { - t.Fatal("expected nil response here") - } -} - -func TestResetKeywordHex(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.ResetKeywordsHex = []string{"|6f 6f 6e 69|"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - resp, err := http.Get("http://www.ooni.io") - if err == nil { - t.Fatal("expected an error here") - } - if strings.Contains(err.Error(), "read: connection reset by peer") == false { - t.Fatal("unexpected error occurred") - } - if resp != nil { - t.Fatal("expected nil response here") - } -} - -func TestHijackDNS(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - resolver := resolver.NewCensoringResolver( - []string{"ooni.io"}, nil, nil, - uncensored.NewClient("https://1.1.1.1/dns-query"), - ) - server, err := resolver.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer server.Shutdown() - policy := newCensoringPolicy() - defer policy.Waive() - policy.HijackDNSAddress = server.PacketConn.LocalAddr().String() - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - reso := &net.Resolver{ - PreferGo: true, - } - addrs, err := reso.LookupHost(context.Background(), "www.ooni.io") - if err == nil { - t.Fatal("expected an error here") - } - if strings.Contains(err.Error(), "no such host") == false { - t.Fatal("unexpected error occurred") - } - if addrs != nil { - t.Fatal("expected nil addrs here") - } -} - -func TestHijackHTTP(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - // Implementation note: this test is complicated by the fact - // that we are running as root and so we're whitelisted. - server := httptest.NewServer(http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(451) - }), - ) - defer server.Close() - policy := newCensoringPolicy() - defer policy.Waive() - pu, err := url.Parse(server.URL) - if err != nil { - t.Fatal(err) - } - policy.HijackHTTPAddress = pu.Host - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - err = shellx.Run(log.Log, "sudo", "-u", "nobody", "--", - "curl", "-sf", "http://example.com") - if err == nil { - t.Fatal("expected an error here") - } - var exitErr *execabs.ExitError - if !errors.As(err, &exitErr) { - t.Fatal("not the error type we expected") - } - if exitErr.ExitCode() != 22 { - t.Fatal("not the exit code we expected") - } -} - -func TestHijackHTTPS(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - // Implementation note: this test is complicated by the fact - // that we are running as root and so we're whitelisted. - server := httptest.NewTLSServer(http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(451) - }), - ) - defer server.Close() - policy := newCensoringPolicy() - defer policy.Waive() - pu, err := url.Parse(server.URL) - if err != nil { - t.Fatal(err) - } - policy.HijackHTTPSAddress = pu.Host - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - err = shellx.Run(log.Log, "sudo", "-u", "nobody", "--", - "curl", "-sf", "https://example.com") - if err == nil { - t.Fatal("expected an error here") - } - t.Log(err) - var exitErr *execabs.ExitError - if !errors.As(err, &exitErr) { - t.Fatal("not the error type we expected") - } - if exitErr.ExitCode() != 60 { - t.Fatal("not the exit code we expected") - } -} diff --git a/internal/cmd/jafar/iptables/iptables_linux.go b/internal/cmd/jafar/iptables/iptables_linux.go deleted file mode 100644 index 09d4c4dd52..0000000000 --- a/internal/cmd/jafar/iptables/iptables_linux.go +++ /dev/null @@ -1,119 +0,0 @@ -//go:build linux - -package iptables - -import ( - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "github.com/ooni/probe-cli/v3/internal/shellx" -) - -type linuxShell struct{} - -func (s *linuxShell) createChains() (err error) { - defer func() { - if recover() != nil { - // JUST KNOW WE'VE BEEN HERE - } - }() - err = shellx.Run(log.Log, "sudo", "iptables", "-N", "JAFAR_INPUT") - runtimex.PanicOnError(err, "cannot create JAFAR_INPUT chain") - err = shellx.Run(log.Log, "sudo", "iptables", "-N", "JAFAR_OUTPUT") - runtimex.PanicOnError(err, "cannot create JAFAR_OUTPUT chain") - err = shellx.Run(log.Log, "sudo", "iptables", "-t", "nat", "-N", "JAFAR_NAT_OUTPUT") - runtimex.PanicOnError(err, "cannot create JAFAR_NAT_OUTPUT chain") - err = shellx.Run(log.Log, "sudo", "iptables", "-I", "OUTPUT", "-j", "JAFAR_OUTPUT") - runtimex.PanicOnError(err, "cannot insert jump to JAFAR_OUTPUT") - err = shellx.Run(log.Log, "sudo", "iptables", "-I", "INPUT", "-j", "JAFAR_INPUT") - runtimex.PanicOnError(err, "cannot insert jump to JAFAR_INPUT") - err = shellx.Run(log.Log, "sudo", "iptables", "-t", "nat", "-I", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT") - runtimex.PanicOnError(err, "cannot insert jump to JAFAR_NAT_OUTPUT") - return nil -} - -func (s *linuxShell) dropIfDestinationEquals(ip string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-d", ip, "-j", "DROP") -} - -func (s *linuxShell) rstIfDestinationEqualsAndIsTCP(ip string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "--proto", "tcp", "-d", ip, - "-j", "REJECT", "--reject-with", "tcp-reset", - ) -} - -func (s *linuxShell) dropIfContainsKeywordHex(keyword string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp", - "--hex-string", keyword, "-j", "DROP", - ) -} - -func (s *linuxShell) dropIfContainsKeyword(keyword string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp", - "--string", keyword, "-j", "DROP", - ) -} - -func (s *linuxShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo", - "kmp", "--hex-string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset", - ) -} - -func (s *linuxShell) rstIfContainsKeywordAndIsTCP(keyword string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo", - "kmp", "--string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset", - ) -} - -func (s *linuxShell) hijackDNS(address string) error { - // Hijack any DNS query, like the Vodafone station does when using the - // secure network feature. Our transparent proxies will use DoT, in order - // to bypass this restriction and avoid routing loop. - return shellx.Run(log.Log, - "sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "udp", - "--dport", "53", "-j", "DNAT", "--to", address, - ) -} - -func (s *linuxShell) hijackHTTPS(address string) error { - // We need to whitelist root otherwise the traffic sent by Jafar - // itself will match the rule and loop. - return shellx.Run(log.Log, - "sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp", - "--dport", "443", "-m", "owner", "!", "--uid-owner", "0", - "-j", "DNAT", "--to", address, - ) -} - -func (s *linuxShell) hijackHTTP(address string) error { - // We need to whitelist root otherwise the traffic sent by Jafar - // itself will match the rule and loop. - return shellx.Run(log.Log, - "sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp", - "--dport", "80", "-m", "owner", "!", "--uid-owner", "0", - "-j", "DNAT", "--to", address, - ) -} - -func (s *linuxShell) waive() error { - shellx.RunQuiet("sudo", "iptables", "-D", "OUTPUT", "-j", "JAFAR_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-D", "INPUT", "-j", "JAFAR_INPUT") - shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-D", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_INPUT") - shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_INPUT") - shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-F", "JAFAR_NAT_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-X", "JAFAR_NAT_OUTPUT") - return nil -} - -func newShell() *linuxShell { - return &linuxShell{} -} diff --git a/internal/cmd/jafar/iptables/iptables_unsupported.go b/internal/cmd/jafar/iptables/iptables_unsupported.go deleted file mode 100644 index 5c45412006..0000000000 --- a/internal/cmd/jafar/iptables/iptables_unsupported.go +++ /dev/null @@ -1,45 +0,0 @@ -//go:build !linux - -package iptables - -import "errors" - -type otherwiseShell struct{} - -func (*otherwiseShell) createChains() error { - return errors.New("not implemented") -} -func (*otherwiseShell) dropIfDestinationEquals(ip string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) rstIfDestinationEqualsAndIsTCP(ip string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) dropIfContainsKeywordHex(keyword string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) dropIfContainsKeyword(keyword string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) rstIfContainsKeywordAndIsTCP(keyword string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) hijackDNS(address string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) hijackHTTPS(address string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) hijackHTTP(address string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) waive() error { - return errors.New("not implemented") -} - -func newShell() *otherwiseShell { - return &otherwiseShell{} -} diff --git a/internal/cmd/jafar/main.go b/internal/cmd/jafar/main.go deleted file mode 100644 index 3a94bfd3b2..0000000000 --- a/internal/cmd/jafar/main.go +++ /dev/null @@ -1,289 +0,0 @@ -// Jafar is a censorship simulation tool used for testing OONI. -package main - -import ( - "encoding/pem" - "errors" - "flag" - "fmt" - "net" - "net/http" - "os" - "os/signal" - "strings" - "syscall" - - "golang.org/x/sys/execabs" - - "github.com/apex/log" - "github.com/apex/log/handlers/cli" - "github.com/miekg/dns" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/badproxy" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/flagx" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/httpproxy" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/iptables" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/resolver" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/tlsproxy" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "github.com/ooni/probe-cli/v3/internal/shellx" -) - -var ( - badProxyAddress *string - badProxyAddressTLS *string - badProxyTLSOutputCA *string - - dnsProxyAddress *string - dnsProxyBlock flagx.StringArray - dnsProxyHijack flagx.StringArray - dnsProxyIgnore flagx.StringArray - - httpProxyAddress *string - httpProxyBlock flagx.StringArray - - iptablesDropIP flagx.StringArray - iptablesDropKeywordHex flagx.StringArray - iptablesDropKeyword flagx.StringArray - iptablesHijackDNSTo *string - iptablesHijackHTTPSTo *string - iptablesHijackHTTPTo *string - iptablesResetIP flagx.StringArray - iptablesResetKeywordHex flagx.StringArray - iptablesResetKeyword flagx.StringArray - - mainCh chan os.Signal - mainCommand *string - mainUser *string - - tag *string - - tlsProxyAddress *string - tlsProxyBlock flagx.StringArray - tlsProxyOutboundPort *string - - uncensoredResolverDoH *string -) - -func init() { - // badProxy - badProxyAddress = flag.String( - "bad-proxy-address", "127.0.0.1:7117", - "Address where to listen for TCP connections", - ) - badProxyAddressTLS = flag.String( - "bad-proxy-address-tls", "127.0.0.1:4114", - "Address where to listen for TLS connections", - ) - badProxyTLSOutputCA = flag.String( - "bad-proxy-tls-output-ca", "badproxy.pem", - "File where to write the CA used by the bad proxy", - ) - - // dnsProxy - dnsProxyAddress = flag.String( - "dns-proxy-address", "127.0.0.1:53", - "Address where the DNS proxy should listen", - ) - flag.Var( - &dnsProxyBlock, "dns-proxy-block", - "Register keyword triggering NXDOMAIN censorship", - ) - flag.Var( - &dnsProxyHijack, "dns-proxy-hijack", - "Register keyword triggering redirection to 127.0.0.1", - ) - flag.Var( - &dnsProxyIgnore, "dns-proxy-ignore", - "Register keyword causing the proxy to ignore the query", - ) - - // httpProxy - httpProxyAddress = flag.String( - "http-proxy-address", "127.0.0.1:80", - "Address where the HTTP proxy should listen", - ) - flag.Var( - &httpProxyBlock, "http-proxy-block", - "Register keyword triggering HTTP 451 censorship", - ) - - // iptables - flag.Var( - &iptablesDropIP, "iptables-drop-ip", - "Drop traffic to the specified IP address", - ) - flag.Var( - &iptablesDropKeywordHex, "iptables-drop-keyword-hex", - "Drop traffic containing the specified keyword in hex", - ) - flag.Var( - &iptablesDropKeyword, "iptables-drop-keyword", - "Drop traffic containing the specified keyword", - ) - iptablesHijackDNSTo = flag.String( - "iptables-hijack-dns-to", "", - "Hijack all DNS UDP traffic to the specified endpoint", - ) - iptablesHijackHTTPSTo = flag.String( - "iptables-hijack-https-to", "", - "Hijack all HTTPS traffic to the specified endpoint", - ) - iptablesHijackHTTPTo = flag.String( - "iptables-hijack-http-to", "", - "Hijack all HTTP traffic to the specified endpoint", - ) - flag.Var( - &iptablesResetIP, "iptables-reset-ip", - "Reset TCP/IP traffic to the specified IP address", - ) - flag.Var( - &iptablesResetKeywordHex, "iptables-reset-keyword-hex", - "Reset TCP/IP traffic containing the specified keyword in hex", - ) - flag.Var( - &iptablesResetKeyword, "iptables-reset-keyword", - "Reset TCP/IP traffic containing the specified keyword", - ) - - // main - mainCh = make(chan os.Signal, 1) - signal.Notify( - mainCh, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, - ) - mainCommand = flag.String("main-command", "", "Optional command to execute") - mainUser = flag.String("main-user", "nobody", "Run command as user") - - // tag - tag = flag.String("tag", "", "Add tag to a specific run") - - // tlsProxy - tlsProxyAddress = flag.String( - "tls-proxy-address", "127.0.0.1:443", - "Address where the TCP+TLS proxy should listen", - ) - flag.Var( - &tlsProxyBlock, "tls-proxy-block", - "Register keyword triggering TLS censorship", - ) - tlsProxyOutboundPort = flag.String( - "tls-proxy-outbound-port", "443", - "The outbound port where requests should be proxied", - ) - - // uncensored - uncensoredResolverDoH = flag.String( - "uncensored-resolver-doh", "https://1.1.1.1/dns-query", - "URL of an hopefully uncensored DoH resolver", - ) -} - -func badProxyStart() net.Listener { - proxy := badproxy.NewCensoringProxy() - listener, err := proxy.Start(*badProxyAddress) - runtimex.PanicOnError(err, "proxy.Start failed") - return listener -} - -func badProxyStartTLS() net.Listener { - proxy := badproxy.NewCensoringProxy() - listener, cert, err := proxy.StartTLS(*badProxyAddressTLS) - runtimex.PanicOnError(err, "proxy.StartTLS failed") - err = os.WriteFile(*badProxyTLSOutputCA, pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: cert.Raw, - }), 0644) - runtimex.PanicOnError(err, "os.WriteFile failed") - return listener -} - -func dnsProxyStart(uncensored *uncensored.Client) *dns.Server { - proxy := resolver.NewCensoringResolver( - dnsProxyBlock, dnsProxyHijack, dnsProxyIgnore, uncensored, - ) - server, err := proxy.Start(*dnsProxyAddress) - runtimex.PanicOnError(err, "proxy.Start failed") - return server -} - -func httpProxyStart(uncensored *uncensored.Client) *http.Server { - proxy := httpproxy.NewCensoringProxy(httpProxyBlock, uncensored) - server, _, err := proxy.Start(*httpProxyAddress) - runtimex.PanicOnError(err, "proxy.Start failed") - return server -} - -func iptablesStart() *iptables.CensoringPolicy { - policy := iptables.NewCensoringPolicy() - // For robustness waive the policy so we start afresh - policy.Waive() - policy.DropIPs = iptablesDropIP - policy.DropKeywordsHex = iptablesDropKeywordHex - policy.DropKeywords = iptablesDropKeyword - policy.HijackDNSAddress = *iptablesHijackDNSTo - policy.HijackHTTPSAddress = *iptablesHijackHTTPSTo - policy.HijackHTTPAddress = *iptablesHijackHTTPTo - policy.ResetIPs = iptablesResetIP - policy.ResetKeywordsHex = iptablesResetKeywordHex - policy.ResetKeywords = iptablesResetKeyword - err := policy.Apply() - runtimex.PanicOnError(err, "policy.Apply failed") - return policy -} - -func tlsProxyStart(uncensored *uncensored.Client) net.Listener { - proxy := tlsproxy.NewCensoringProxy(tlsProxyBlock, uncensored, *tlsProxyOutboundPort) - listener, err := proxy.Start(*tlsProxyAddress) - runtimex.PanicOnError(err, "proxy.Start failed") - return listener -} - -func newUncensoredClient() *uncensored.Client { - return uncensored.NewClient(*uncensoredResolverDoH) -} - -func mustx(err error, message string, osExit func(int)) { - if err != nil { - var ( - exitcode = 1 - exiterr *execabs.ExitError - ) - if errors.As(err, &exiterr) { - exitcode = exiterr.ExitCode() - } - log.Errorf("%s", message) - osExit(exitcode) - } -} - -func main() { - flag.Parse() - // TODO(bassosimone): we may want a verbose flag - log.SetLevel(log.InfoLevel) - log.SetHandler(cli.Default) - log.Infof("jafar command line: [%s]", strings.Join(os.Args, ", ")) - log.Infof("jafar tag: %s", *tag) - uncensoredClient := newUncensoredClient() - defer uncensoredClient.CloseIdleConnections() - badlistener := badProxyStart() - defer badlistener.Close() - badtlslistener := badProxyStartTLS() - defer badtlslistener.Close() - dnsproxy := dnsProxyStart(uncensoredClient) - defer dnsproxy.Shutdown() - httpproxy := httpProxyStart(uncensoredClient) - defer httpproxy.Close() - tlslistener := tlsProxyStart(uncensoredClient) - defer tlslistener.Close() - policy := iptablesStart() - var err error - if *mainCommand != "" { - err = shellx.RunCommandLine(log.Log, fmt.Sprintf( - "sudo -u '%s' -- %s", *mainUser, *mainCommand, - )) - } else { - <-mainCh - } - policy.Waive() - mustx(err, "subcommand failed", os.Exit) -} diff --git a/internal/cmd/jafar/main_test.go b/internal/cmd/jafar/main_test.go deleted file mode 100644 index 879cf5bf2a..0000000000 --- a/internal/cmd/jafar/main_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package main - -import ( - "errors" - "os" - "runtime" - "testing" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/iptables" - "github.com/ooni/probe-cli/v3/internal/shellx" -) - -func ensureWeStartOverWithIPTables() { - iptables.NewCensoringPolicy().Waive() -} - -func TestNoCommand(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - if runtime.GOOS != "linux" { - t.Skip("skip test on non Linux systems") - } - ensureWeStartOverWithIPTables() - *dnsProxyAddress = "127.0.0.1:0" - *httpProxyAddress = "127.0.0.1:0" - *tlsProxyAddress = "127.0.0.1:0" - go func() { - mainCh <- os.Interrupt - }() - main() -} - -func TestWithCommand(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - if runtime.GOOS != "linux" { - t.Skip("skip test on non Linux systems") - } - ensureWeStartOverWithIPTables() - *dnsProxyAddress = "127.0.0.1:0" - *httpProxyAddress = "127.0.0.1:0" - *tlsProxyAddress = "127.0.0.1:0" - *mainCommand = "whoami" - defer func() { - *mainCommand = "" - }() - main() -} - -func TestMustx(t *testing.T) { - t.Run("with no error", func(t *testing.T) { - var called int - mustx(nil, "", func(int) { - called++ - }) - if called != 0 { - t.Fatal("should not happen") - } - }) - - t.Run("with non-exit-code error", func(t *testing.T) { - var ( - called int - exitcode int - ) - mustx(errors.New("antani"), "", func(ec int) { - called++ - exitcode = ec - }) - if called != 1 { - t.Fatal("not called?!") - } - if exitcode != 1 { - t.Fatal("unexpected exitcode value") - } - }) - - t.Run("with exit-code error", func(t *testing.T) { - var ( - called int - exitcode int - ) - err := shellx.Run(log.Log, "curl", "-sf", "") // cause exitcode == 3 - mustx(err, "", func(ec int) { - called++ - exitcode = ec - }) - if called != 1 { - t.Fatal("not called?!") - } - if exitcode != 3 { - t.Fatal("unexpected exitcode value") - } - }) -} diff --git a/internal/cmd/jafar/resolver/resolver.go b/internal/cmd/jafar/resolver/resolver.go deleted file mode 100644 index 44eba27cb0..0000000000 --- a/internal/cmd/jafar/resolver/resolver.go +++ /dev/null @@ -1,134 +0,0 @@ -// Package resolver contains a censoring DNS resolver. Most queries are -// answered without censorship, but selected queries could either be -// discarded or replied to with a bogon or NXDOMAIN answer. -package resolver - -import ( - "context" - "net" - "strings" - - "github.com/miekg/dns" -) - -// Resolver resolves domain names. -type Resolver interface { - LookupHost(ctx context.Context, hostname string) ([]string, error) -} - -// CensoringResolver is a censoring resolver. -type CensoringResolver struct { - blocked []string - hijacked []string - ignored []string - lookupHost func(ctx context.Context, host string) ([]string, error) -} - -// NewCensoringResolver creates a new CensoringResolver instance using -// the specified list of keywords to censor. blocked is the list of -// keywords that trigger NXDOMAIN if they appear in a query. hijacked -// is similar but redirects to 127.0.0.1, where the transparent HTTP -// and TLS proxies will pick them up. dnsNetwork and dnsAddress are the -// settings to configure the upstream, non censored DNS. -func NewCensoringResolver( - blocked, hijacked, ignored []string, uncensored Resolver, -) *CensoringResolver { - return &CensoringResolver{ - blocked: blocked, - hijacked: hijacked, - ignored: ignored, - lookupHost: uncensored.LookupHost, - } -} - -func (r *CensoringResolver) roundtrip(rw dns.ResponseWriter, req *dns.Msg) { - name := req.Question[0].Name - addrs, err := r.lookupHost(context.Background(), name) - var ips []net.IP - if err == nil { - for _, addr := range addrs { - if ip := net.ParseIP(addr); ip != nil { - ips = append(ips, ip) - } - } - } - r.reply(rw, req, ips) -} - -func (r *CensoringResolver) reply( - rw dns.ResponseWriter, req *dns.Msg, ips []net.IP, -) { - m := new(dns.Msg) - m.Compress = true - m.MsgHdr.RecursionAvailable = true - m.SetReply(req) - for _, ip := range ips { - ipv6 := strings.Contains(ip.String(), ":") - if !ipv6 && req.Question[0].Qtype == dns.TypeA { - m.Answer = append(m.Answer, &dns.A{ - Hdr: dns.RR_Header{ - Name: req.Question[0].Name, - Rrtype: dns.TypeA, - Class: dns.ClassINET, - Ttl: 0, - }, - A: ip, - }) - } - } - if m.Answer == nil { - m.SetRcode(req, dns.RcodeNameError) - } - rw.WriteMsg(m) -} - -func (r *CensoringResolver) failure(rw dns.ResponseWriter, req *dns.Msg) { - m := new(dns.Msg) - m.Compress = true - m.MsgHdr.RecursionAvailable = true - m.SetRcode(req, dns.RcodeServerFailure) - rw.WriteMsg(m) -} - -// ServeDNS serves a DNS request -func (r *CensoringResolver) ServeDNS(rw dns.ResponseWriter, req *dns.Msg) { - if len(req.Question) < 1 { - r.failure(rw, req) - return - } - name := req.Question[0].Name - for _, pattern := range r.blocked { - if strings.Contains(name, pattern) { - r.reply(rw, req, nil) - return - } - } - for _, pattern := range r.hijacked { - if strings.Contains(name, pattern) { - r.reply(rw, req, []net.IP{net.IPv4(127, 0, 0, 1)}) - return - } - } - for _, pattern := range r.ignored { - if strings.Contains(name, pattern) { - return - } - } - r.roundtrip(rw, req) -} - -// Start starts the DNS resolver -func (r *CensoringResolver) Start(address string) (*dns.Server, error) { - packetconn, err := net.ListenPacket("udp", address) - if err != nil { - return nil, err - } - server := &dns.Server{ - Addr: address, - Handler: r, - Net: "udp", - PacketConn: packetconn, - } - go server.ActivateAndServe() - return server, nil -} diff --git a/internal/cmd/jafar/resolver/resolver_test.go b/internal/cmd/jafar/resolver/resolver_test.go deleted file mode 100644 index 103344dda0..0000000000 --- a/internal/cmd/jafar/resolver/resolver_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package resolver - -import ( - "strings" - "testing" - - "github.com/miekg/dns" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" -) - -func TestPass(t *testing.T) { - server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil) - checkrequest(t, server, "example.com", "success", nil) - killserver(t, server) -} - -func TestBlock(t *testing.T) { - server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil) - checkrequest(t, server, "mia-ps.ooni.io", "blocked", nil) - killserver(t, server) -} - -func TestRedirect(t *testing.T) { - server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil) - checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", nil) - killserver(t, server) -} - -func TestIgnore(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - server := newresolver(t, nil, nil, []string{"ooni.nu"}) - iotimeout := "i/o timeout" - checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", &iotimeout) - killserver(t, server) -} - -func TestLookupFailure(t *testing.T) { - server := newresolver(t, nil, nil, nil) - // we should receive same response as when we're blocked - checkrequest(t, server, "example.antani", "blocked", nil) - killserver(t, server) -} - -func TestFailureNoQuestion(t *testing.T) { - resolver := NewCensoringResolver( - nil, nil, nil, uncensored.NewClient("https://1.1.1.1/dns-query"), - ) - resolver.ServeDNS(&fakeResponseWriter{t: t}, new(dns.Msg)) -} - -func TestListenFailure(t *testing.T) { - resolver := NewCensoringResolver( - nil, nil, nil, uncensored.NewClient("https://1.1.1.1/dns-query"), - ) - server, err := resolver.Start("8.8.8.8:53") - if err == nil { - t.Fatal("expected an error here") - } - if server != nil { - t.Fatal("expected nil server here") - } -} - -func newresolver(t *testing.T, blocked, hijacked, ignored []string) *dns.Server { - resolver := NewCensoringResolver( - blocked, hijacked, ignored, - uncensored.NewClient("https://1.1.1.1/dns-query"), - ) - server, err := resolver.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return server -} - -func killserver(t *testing.T, server *dns.Server) { - err := server.Shutdown() - if err != nil { - t.Fatal(err) - } -} - -func checkrequest( - t *testing.T, server *dns.Server, host string, expectStatus string, - expectErrorSuffix *string, -) { - address := server.PacketConn.LocalAddr().String() - query := newquery(host) - reply, err := dns.Exchange(query, address) - if err != nil { - if expectErrorSuffix != nil && - strings.HasSuffix(err.Error(), *expectErrorSuffix) { - return - } - t.Fatal(err) - } - switch expectStatus { - case "success": - checksuccess(t, reply) - case "hijacked": - checkhijacked(t, reply) - case "blocked": - checkblocked(t, reply) - default: - panic("unexpected value") - } -} - -func checksuccess(t *testing.T, reply *dns.Msg) { - if reply.Rcode != dns.RcodeSuccess { - t.Fatal("unexpected rcode", reply.Rcode) - } - if len(reply.Answer) < 1 { - t.Fatal("too few answers") - } - for _, answer := range reply.Answer { - if rr, ok := answer.(*dns.A); ok { - if rr.A.String() == "127.0.0.1" { - t.Fatal("unexpected hijacked response here") - } - } - } -} - -func checkhijacked(t *testing.T, reply *dns.Msg) { - if reply.Rcode != dns.RcodeSuccess { - t.Fatal("unexpected rcode") - } - if len(reply.Answer) < 1 { - t.Fatal("too few answers") - } - for _, answer := range reply.Answer { - if rr, ok := answer.(*dns.A); ok { - if rr.A.String() != "127.0.0.1" { - t.Fatal("unexpected non-hijacked response here") - } - } - } -} - -func checkblocked(t *testing.T, reply *dns.Msg) { - if reply.Rcode != dns.RcodeNameError { - t.Fatal("unexpected rcode") - } - if len(reply.Answer) >= 1 { - t.Fatal("too many answers") - } -} - -func newquery(name string) *dns.Msg { - query := new(dns.Msg) - query.Id = dns.Id() - query.RecursionDesired = true - query.Question = append(query.Question, dns.Question{ - Name: dns.Fqdn(name), - Qclass: dns.ClassINET, - Qtype: dns.TypeA, - }) - return query -} - -type fakeResponseWriter struct { - dns.ResponseWriter - t *testing.T -} - -func (rw *fakeResponseWriter) WriteMsg(m *dns.Msg) error { - if m.Rcode != dns.RcodeServerFailure { - rw.t.Fatal("unexpected rcode") - } - return nil -} diff --git a/internal/cmd/jafar/tlsproxy/tlsproxy.go b/internal/cmd/jafar/tlsproxy/tlsproxy.go deleted file mode 100644 index 401333da8a..0000000000 --- a/internal/cmd/jafar/tlsproxy/tlsproxy.go +++ /dev/null @@ -1,200 +0,0 @@ -// Package tlsproxy contains a censoring TLS proxy. Most traffic is passed -// through using the SNI to choose the hostname to connect to. Specific offending -// SNIs are censored by returning a TLS alert to the client. -package tlsproxy - -import ( - "context" - "crypto/tls" - "errors" - "net" - "strings" - "sync" - - "github.com/apex/log" -) - -// Dialer establishes network connections -type Dialer interface { - DialContext(ctx context.Context, network, address string) (net.Conn, error) -} - -// CensoringProxy is a censoring TLS proxy -type CensoringProxy struct { - keywords []string - dial func(network, address string) (net.Conn, error) - outboundPort string -} - -// NewCensoringProxy creates a new CensoringProxy instance using -// the specified list of keywords to censor. keywords is the list -// of keywords that trigger censorship if any of them appears in -// the SNI record of a ClientHello. dnsNetwork and dnsAddress are -// settings to configure the upstream, non censored DNS. -func NewCensoringProxy( - keywords []string, uncensored Dialer, outboundPort string, -) *CensoringProxy { - return &CensoringProxy{ - keywords: keywords, - dial: func(network, address string) (net.Conn, error) { - return uncensored.DialContext(context.Background(), network, address) - }, - outboundPort: outboundPort, - } -} - -// handshakeReader is a hack to perform the initial part of the -// TLS handshake so to know the SNI and then replay the bytes of -// this initial part of the handshake with the server. -type handshakeReader struct { - net.Conn - incoming []byte -} - -// Read saves the initial bytes of the handshake such that later -// we can replay the handshake with the real TLS server. -func (c *handshakeReader) Read(b []byte) (int, error) { - count, err := c.Conn.Read(b) - if err == nil { - c.incoming = append(c.incoming, b[:count]...) - } - return count, err -} - -// Write prevents writing on the real connection -func (c *handshakeReader) Write(b []byte) (int, error) { - return 0, errors.New("cannot write on this connection") -} - -// forward forwards left traffic to right -func forward(wg *sync.WaitGroup, left, right net.Conn) { - data := make([]byte, 1<<18) - for { - n, err := left.Read(data) - if err != nil { - break - } - if _, err = right.Write(data[:n]); err != nil { - break - } - } - wg.Done() -} - -// reset closes the connection with a RST segment -func reset(conn net.Conn) { - if tc, ok := conn.(*net.TCPConn); ok { - tc.SetLinger(0) - } - conn.Close() -} - -// alertclose sends a TLS alert and then closes the connection -func alertclose(conn net.Conn) { - alertdata := []byte{ - 21, // alert - 3, // version[0] - 3, // version[1] - 0, // length[0] - 2, // length[1] - 2, // fatal - 80, // internal error - } - conn.Write(alertdata) - conn.Close() -} - -// getsni attempts the handshakeReader hack to obtain the SNI by reading -// the beginning of the TLS handshake. On success a nonempty SNI string -// is returned. Otherwise we cannot distinguish between the absence of a -// SNI and any other reading network error that may have occurred. -func getsni(conn *handshakeReader) string { - var ( - sni string - mutex sync.Mutex // just for safety - ) - tls.Server(conn, &tls.Config{ - GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - mutex.Lock() - sni = info.ServerName - mutex.Unlock() - return nil, errors.New("tlsproxy: we can't really continue handshake") - }, - }).Handshake() - return sni -} - -func (p *CensoringProxy) connectingToMyself(conn net.Conn) bool { - local := conn.LocalAddr().String() - localAddr, _, localErr := net.SplitHostPort(local) - remote := conn.RemoteAddr().String() - remoteAddr, _, remoteErr := net.SplitHostPort(remote) - return localErr != nil || remoteErr != nil || localAddr == remoteAddr -} - -// handle implements the TLS SNI proxy -func (p *CensoringProxy) handle(clientconn net.Conn) { - hr := &handshakeReader{Conn: clientconn} - sni := getsni(hr) - if sni == "" { - log.Warn("tlsproxy: network failure or SNI not provided") - reset(clientconn) - return - } - for _, pattern := range p.keywords { - if strings.Contains(sni, pattern) { - log.Warnf("tlsproxy: reject SNI by policy: %s", sni) - alertclose(clientconn) - return - } - } - serverconn, err := p.dial("tcp", net.JoinHostPort(sni, p.outboundPort)) - if err != nil { - log.WithError(err).Warn("tlsproxy: p.dial failed") - alertclose(clientconn) - return - } - if p.connectingToMyself(serverconn) { - log.Warn("tlsproxy: connecting to myself") - alertclose(clientconn) - return - } - if _, err := serverconn.Write(hr.incoming); err != nil { - log.WithError(err).Warn("tlsproxy: serverconn.Write failed") - alertclose(clientconn) - return - } - log.Debugf("tlsproxy: routing for %s", sni) - defer clientconn.Close() - defer serverconn.Close() - var wg sync.WaitGroup - wg.Add(2) - go forward(&wg, clientconn, serverconn) - go forward(&wg, serverconn, clientconn) - wg.Wait() -} - -func (p *CensoringProxy) run(listener net.Listener) { - for { - conn, err := listener.Accept() - if err != nil && strings.Contains( - err.Error(), "use of closed network connection") { - return - } - if err == nil { - // It's difficult to make accept fail, so restructure - // the code such that we enter into the happy path - go p.handle(conn) - } - } -} - -// Start starts the censoring proxy. -func (p *CensoringProxy) Start(address string) (net.Listener, error) { - listener, err := net.Listen("tcp", address) - if err != nil { - return nil, err - } - go p.run(listener) - return listener, nil -} diff --git a/internal/cmd/jafar/tlsproxy/tlsproxy_test.go b/internal/cmd/jafar/tlsproxy/tlsproxy_test.go deleted file mode 100644 index 0887ca4efd..0000000000 --- a/internal/cmd/jafar/tlsproxy/tlsproxy_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package tlsproxy - -import ( - "crypto/tls" - "errors" - "net" - "sync" - "testing" - - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" -) - -func TestPass(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), true, &tls.Config{ - ServerName: "example.com", - }) - killproxy(t, listener) -} - -func TestBlock(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "api.ooni.io", - }) - killproxy(t, listener) -} - -func TestNoSNI(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "", - }) - killproxy(t, listener) -} - -func TestInvalidDomain(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "antani.local", - }) - killproxy(t, listener) -} - -func TestFailHandshake(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "expired.badssl.com", - }) - killproxy(t, listener) -} - -func TestFailConnectingToSelf(t *testing.T) { - proxy := &CensoringProxy{ - dial: func(network string, address string) (net.Conn, error) { - return &mockedConnWriteError{}, nil - }, - } - listener, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - if listener == nil { - t.Fatal("expected non nil listener here") - } - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "www.google.com", - }) - killproxy(t, listener) -} - -func TestFailWriteAfterConnect(t *testing.T) { - proxy := &CensoringProxy{ - dial: func(network string, address string) (net.Conn, error) { - return &mockedConnWriteError{ - // must be different or it refuses connecting to self - localIP: net.IPv4(127, 0, 0, 1), - remoteIP: net.IPv4(127, 0, 0, 2), - }, nil - }, - } - listener, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - if listener == nil { - t.Fatal("expected non nil listener here") - } - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "www.google.com", - }) - killproxy(t, listener) -} - -func TestListenError(t *testing.T) { - proxy := NewCensoringProxy( - []string{""}, uncensored.NewClient("https://1.1.1.1/dns-query"), "443", - ) - listener, err := proxy.Start("8.8.8.8:80") - if err == nil { - t.Fatal("expected an error here") - } - if listener != nil { - t.Fatal("expected nil listener here") - } -} - -func newproxy(t *testing.T, blocked string) net.Listener { - proxy := NewCensoringProxy( - []string{blocked}, uncensored.NewClient("https://1.1.1.1/dns-query"), "443", - ) - listener, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return listener -} - -func killproxy(t *testing.T, listener net.Listener) { - err := listener.Close() - if err != nil { - t.Fatal(err) - } -} - -func checkdialtls( - t *testing.T, proxyAddr string, expectSuccess bool, config *tls.Config, -) { - conn, err := tls.Dial("tcp", proxyAddr, config) - if err != nil && expectSuccess { - t.Fatal(err) - } - if err == nil && !expectSuccess { - t.Fatal("expected failure here") - } - if conn == nil && expectSuccess { - t.Fatal("expected actionable conn") - } - if conn != nil && !expectSuccess { - t.Fatal("expected nil conn") - } - if conn != nil { - conn.Close() - } -} - -type mockedConnWriteError struct { - net.Conn - localIP net.IP - remoteIP net.IP -} - -func (c *mockedConnWriteError) Write(b []byte) (int, error) { - return 0, errors.New("cannot write sorry") -} - -func (c *mockedConnWriteError) LocalAddr() net.Addr { - return &net.TCPAddr{ - IP: c.localIP, - } -} - -func (c *mockedConnWriteError) RemoteAddr() net.Addr { - return &net.TCPAddr{ - IP: c.remoteIP, - } -} - -func TestForwardWriteError(t *testing.T) { - var wg sync.WaitGroup - wg.Add(1) - forward(&wg, &mockedConnReadOkay{}, &mockedConnWriteError{}) -} - -type mockedConnReadOkay struct { - net.Conn -} - -func (c *mockedConnReadOkay) Read(b []byte) (int, error) { - return len(b), nil -} diff --git a/internal/cmd/jafar/uncensored/uncensored.go b/internal/cmd/jafar/uncensored/uncensored.go deleted file mode 100644 index fcb51a7e4a..0000000000 --- a/internal/cmd/jafar/uncensored/uncensored.go +++ /dev/null @@ -1,72 +0,0 @@ -// Package uncensored contains code used by Jafar to evade its own -// censorship efforts by taking alternate routes. -package uncensored - -import ( - "context" - "errors" - "net" - "net/http" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// Client is DNS, HTTP, and TCP client. -type Client struct { - dnsClient model.Resolver - httpTransport model.HTTPTransport - dialer model.Dialer -} - -// NewClient creates a new Client. -func NewClient(resolverURL string) *Client { - dnsClient := netxlite.NewParallelDNSOverHTTPSResolver(log.Log, resolverURL) - return &Client{ - dnsClient: dnsClient, - httpTransport: netxlite.NewHTTPTransportWithResolver(log.Log, dnsClient), - dialer: netxlite.NewDialerWithResolver(log.Log, dnsClient), - } -} - -// Address implements Resolver.Address -func (c *Client) Address() string { - return c.dnsClient.Address() -} - -// LookupHost implements Resolver.LookupHost -func (c *Client) LookupHost(ctx context.Context, domain string) ([]string, error) { - return c.dnsClient.LookupHost(ctx, domain) -} - -// LookupHTTPS implements model.Resolver.LookupHTTPS. -func (c *Client) LookupHTTPS(ctx context.Context, domain string) (*model.HTTPSSvc, error) { - return nil, errors.New("not implemented") -} - -// LookupNS implements model.Resolver.LookupNS. -func (c *Client) LookupNS(ctx context.Context, domain string) ([]*net.NS, error) { - return nil, errors.New("not implemented") -} - -// Network implements Resolver.Network -func (c *Client) Network() string { - return c.dnsClient.Network() -} - -// DialContext implements Dialer.DialContext -func (c *Client) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - return c.dialer.DialContext(ctx, network, address) -} - -// CloseIdleConnections implement HTTPRoundTripper.CloseIdleConnections -func (c *Client) CloseIdleConnections() { - c.dnsClient.CloseIdleConnections() - c.httpTransport.CloseIdleConnections() -} - -// RoundTrip implement HTTPRoundTripper.RoundTrip -func (c *Client) RoundTrip(req *http.Request) (*http.Response, error) { - return c.httpTransport.RoundTrip(req) -} diff --git a/internal/cmd/jafar/uncensored/uncensored_test.go b/internal/cmd/jafar/uncensored/uncensored_test.go deleted file mode 100644 index d03f229ad3..0000000000 --- a/internal/cmd/jafar/uncensored/uncensored_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package uncensored - -import ( - "bytes" - "context" - "net/http" - "net/url" - "testing" - - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestNewClient(t *testing.T) { - client := NewClient("https://1.1.1.1/dns-query") - defer client.CloseIdleConnections() - if client.Address() != "https://1.1.1.1/dns-query" { - t.Fatal("invalid address") - } - if client.Network() != "doh" { - t.Fatal("invalid network") - } - ctx := context.Background() - addrs, err := client.LookupHost(ctx, "dns.google") - if err != nil { - t.Fatal(err) - } - var quad8, two8two4 bool - for _, addr := range addrs { - quad8 = quad8 || (addr == "8.8.8.8") - two8two4 = two8two4 || (addr == "8.8.4.4") - } - if quad8 != true && two8two4 != true { - t.Fatal("invalid response") - } - conn, err := client.DialContext(ctx, "tcp", "8.8.8.8:853") - if err != nil { - t.Fatal(err) - } - defer conn.Close() - resp, err := client.RoundTrip(&http.Request{ - Method: "GET", - URL: &url.URL{ - Scheme: "https", - Host: "www.google.com", - Path: "/humans.txt", - }, - Header: http.Header{}, - }) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatal("invalid status-code") - } - data, err := netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - if !bytes.HasPrefix(data, []byte("Google is built by a large team")) { - t.Fatal("not the expected body") - } -} diff --git a/internal/cmd/miniooni/README.md b/internal/cmd/miniooni/README.md index c4ab7b6666..0f4d96299d 100644 --- a/internal/cmd/miniooni/README.md +++ b/internal/cmd/miniooni/README.md @@ -2,8 +2,7 @@ This directory contains the source code of a simple CLI client that we use for research as well as for running QA scripts. We designed this tool -to have a CLI similar to MK and OONI Probe v2.x to ease running Jafar -scripts that check whether these tools behave similarly. Perfect backwards +to have a CLI similar to MK and OONI Probe v2.x to ease A/B testing. Perfect backwards compatibility was not a design goal for miniooni. Rather, we aimed to have as little conflict as possible, such that we can run side-by-side QA checks. diff --git a/internal/cmd/tinyjafar/README.md b/internal/cmd/tinyjafar/README.md new file mode 100644 index 0000000000..9b44ad4ceb --- /dev/null +++ b/internal/cmd/tinyjafar/README.md @@ -0,0 +1,113 @@ +# internal/cmd/tinyjafar + +This directory builds a program you can use to provoke simple network +interference conditions, as described in detail below. + +To build, use: + +```console +go build -v ./internal/cmd/tinyjafar +``` + +Any requirement that applies to building OONI Probe also applies to +building this small helper program, since they use the same base library. + +The command line interface is backwards compatible with the one +implemented by [the original jafar](https://github.com/ooni/probe-cli/tree/v3.18.1/internal/cmd/jafar) +except that `tinyjafar` only supports iptables flags. + +To use this tool, you must be on Linux and have iptables installed. We do not +use this tool for QA, but it is mentioned in [tutorials](../../../internal/tutorial/). + +## Drop traffic towards a given IP address + +In one console, run: + +```console +./tinyjafar -iptables-drop-ip 130.192.16.171 +``` + +The program will run some `iptables` commands showing each command +it runs. These commands configure `iptables` to block some internet traffic +and the blocking will stay in place until you interrupt +`tinyjafar` using Ctrl-C. When existing, `tinyjafar` will +undo all the commands it executed when starting up. + +While `tinyjafar` is running, in another console try this command: + +```console +curl -v https://nexa.polito.it/ +``` + +If the IP address has not changed since writing this README, the +`curl` command should eventually timeout when connecting. + +## Drop packets containing an hex sequence + +```console +./tinyjafar -iptables-drop-keyword-hex "|07 65 78 61 6d 70 6c 65 03 63 6f 6d|" +``` + +and + +```console +dig @8.8.8.8 www.example.com +``` + +The `tinyjafar` invocation drops DNS queries for `www.example.com`. + +## Drop packets containing a string + +```console +./tinyjafar -iptables-drop-keyword ooni.org +``` + +and + +```console +curl -v https://ooni.org/ +``` + +We expect cURL to timeout during the TLS handshake since we're +blocking the string that appears in the SNI field. + +## Preventing TCP-connecting to a host + +```console +./tinyjafar -iptables-reset-ip 130.192.16.171 +``` + +and + +```console +curl -v https://nexa.polito.it/ +``` + +This should fail with "connection refused". + +## Resetting a TCP connection containing an hex pattern + +```console +./tinyjafar -iptables-reset-keyword-hex "|6F 6F 6E 69|" +``` + +and + +```console +curl -v https://ooni.org/ +``` + +This should reset the TCP connection because the TLS Client Hello +contains "ooni" (`6F 6F 6E 69` in hex). + +## Resetting a TCP connection containing a string pattern + +`console +./tinyjafar -iptables-reset-keyword ooni +``` + +and + +```console +curl -v https://ooni.org/ +``` diff --git a/internal/cmd/tinyjafar/main.go b/internal/cmd/tinyjafar/main.go new file mode 100644 index 0000000000..b962d2c890 --- /dev/null +++ b/internal/cmd/tinyjafar/main.go @@ -0,0 +1,188 @@ +// Command tinyjafar implements a subset of the CLI flags of the original jafar tool. Because several +// tutorials mention some jafar commands, we want to have a tiny tool to support exploration. +package main + +import ( + "flag" + "fmt" + "io" + "os" + "os/signal" + "sync/atomic" + "syscall" + + "github.com/apex/log" + "github.com/google/shlex" + "github.com/ooni/probe-cli/v3/internal/flagx" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// config contains tinyjafar's configuration. +type config struct { + dropIP flagx.StringArray + dropKeywordHex flagx.StringArray + dropKeyword flagx.StringArray + dryRun bool + resetIP flagx.StringArray + resetKeywordHex flagx.StringArray + resetKeyword flagx.StringArray +} + +func (cfg *config) initFlags(fset *flag.FlagSet) { + fset.Var(&cfg.dropIP, "iptables-drop-ip", "Drop traffic to the specified IP address") + fset.Var(&cfg.dropKeywordHex, "iptables-drop-keyword-hex", "Drop traffic containing the specified keyword in hex") + fset.Var(&cfg.dropKeyword, "iptables-drop-keyword", "Drop traffic containing the specified keyword") + fset.BoolVar(&cfg.dryRun, "dry-run", false, "print which commands we would execute") + fset.Var(&cfg.resetIP, "iptables-reset-ip", "Reset TCP/IP traffic to the specified IP address") + fset.Var(&cfg.resetKeywordHex, "iptables-reset-keyword-hex", "Reset TCP/IP traffic containing the specified keyword in hex") + fset.Var(&cfg.resetKeyword, "iptables-reset-keyword", "Reset TCP/IP traffic containing the specified keyword") +} + +// cmd is a cmd to execute +type cmd struct { + argv []string +} + +// cmdSet contains the commands to execute. The zero value is invalid +// and you must construct using the [newCmdSet] factory. +type cmdSet struct { + setup []*cmd + cleanup []*cmd +} + +func newCmdSet() *cmdSet { + c := &cmdSet{} + + c.addSetupCmd("iptables -N JAFAR_INPUT") + c.addSetupCmd("iptables -N JAFAR_OUTPUT") + c.addSetupCmd("iptables -t nat -N JAFAR_NAT_OUTPUT") + c.addSetupCmd("iptables -I OUTPUT -j JAFAR_OUTPUT") + c.addSetupCmd("iptables -I INPUT -j JAFAR_INPUT") + c.addSetupCmd("iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT") + + addCleanupCmd := func(argv string) { + c.cleanup = append(c.cleanup, &cmd{runtimex.Try1(shlex.Split(argv))}) + } + + addCleanupCmd("iptables -D OUTPUT -j JAFAR_OUTPUT") + addCleanupCmd("iptables -D INPUT -j JAFAR_INPUT") + addCleanupCmd("iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT") + addCleanupCmd("iptables -F JAFAR_INPUT") + addCleanupCmd("iptables -X JAFAR_INPUT") + addCleanupCmd("iptables -F JAFAR_OUTPUT") + addCleanupCmd("iptables -X JAFAR_OUTPUT") + addCleanupCmd("iptables -t nat -F JAFAR_NAT_OUTPUT") + addCleanupCmd("iptables -t nat -X JAFAR_NAT_OUTPUT") + + return c +} + +func (c *cmdSet) addSetupCmd(argv string) { + c.setup = append(c.setup, &cmd{runtimex.Try1(shlex.Split(argv))}) +} + +func (c *cmdSet) handleDropIP(cfg *config) { + for _, ipAddr := range cfg.dropIP { + c.addSetupCmd(fmt.Sprintf("iptables -A JAFAR_OUTPUT -d '%s' -j DROP", ipAddr)) + } +} + +func (c *cmdSet) handleDropKeywordHex(cfg *config) { + for _, keyword := range cfg.dropKeywordHex { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT -m string --algo kmp --hex-string '%s' -j DROP", keyword)) + } +} + +func (c *cmdSet) handleDropKeyword(cfg *config) { + for _, keyword := range cfg.dropKeyword { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT -m string --algo kmp --string '%s' -j DROP", + keyword, + )) + } +} + +func (c *cmdSet) handleResetIP(cfg *config) { + for _, ipAddr := range cfg.resetIP { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT --proto tcp -d '%s' -j REJECT --reject-with tcp-reset", + ipAddr, + )) + } +} + +func (c *cmdSet) handleResetKeywordHex(cfg *config) { + for _, keyword := range cfg.resetKeywordHex { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT -m string --proto tcp --algo kmp --hex-string '%s' -j REJECT --reject-with tcp-reset", + keyword, + )) + } +} + +func (c *cmdSet) handleResetKeyword(cfg *config) { + for _, keyword := range cfg.resetKeyword { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT -m string --proto tcp --algo kmp --string '%s' -j REJECT --reject-with tcp-reset", + keyword, + )) + } +} + +func main() { + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGINT) + mainWithArgs(os.Stdout, sigChan, os.Args[1:]...) +} + +var ( + returnImmediately = &atomic.Bool{} + mainWithArgsCalled = &atomic.Int64{} +) + +func mainWithArgs(writer io.Writer, sigChan <-chan os.Signal, args ...string) { + if returnImmediately.Load() { + mainWithArgsCalled.Add(1) + return + } + + cfg := &config{} + fset := flag.NewFlagSet("tinyjafar", flag.ExitOnError) + cfg.initFlags(fset) + + fset.Parse(args) + + cs := newCmdSet() + cs.handleDropIP(cfg) + cs.handleDropKeywordHex(cfg) + cs.handleDropKeyword(cfg) + cs.handleResetIP(cfg) + cs.handleResetKeywordHex(cfg) + cs.handleResetKeyword(cfg) + + // with -dry-run, we're just going to print the commands we'd execute + dryShellRun := func(logger model.Logger, command string, args ...string) error { + _, err := fmt.Fprintf(writer, "+ %s\n", shellx.QuotedCommandLineUnsafe(command, args...)) + return err + } + var runSelector = map[bool]func(logger model.Logger, command string, args ...string) error{ + true: dryShellRun, + false: shellx.Run, + } + runx := runSelector[cfg.dryRun] + + for _, cmd := range cs.setup { + runtimex.Try0(runx(log.Log, cmd.argv[0], cmd.argv[1:]...)) + } + + fmt.Fprintf(writer, "\nUse Ctrl-C to terminate\n\n") + <-sigChan + + for _, cmd := range cs.cleanup { + // ignoring the return value here is intentional to avoid interrupting the cleanup midway + _ = runx(log.Log, cmd.argv[0], cmd.argv[1:]...) + } +} diff --git a/internal/cmd/tinyjafar/main_test.go b/internal/cmd/tinyjafar/main_test.go new file mode 100644 index 0000000000..bcff5dc9f9 --- /dev/null +++ b/internal/cmd/tinyjafar/main_test.go @@ -0,0 +1,254 @@ +package main + +import ( + "os" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestMain(t *testing.T) { + t.Run("test the main function with ReturnImmediately", func(t *testing.T) { + defer returnImmediately.Store(false) + returnImmediately.Store(true) + main() + if mainWithArgsCalled.Load() != 1 { + t.Fatal("main did not call mainWithArguments") + } + }) + + // testcase is a test case for this function + type testcase struct { + // name is the test case name + name string + + // args contains the arguments passed to the command line + args []string + + // expect contains the expected program output + expect []string + } + + testcases := []testcase{ + { + name: "without any command line argument", + args: []string{}, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-drop-ip", + args: []string{ + "-iptables-drop-ip", "130.192.16.171", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -A JAFAR_OUTPUT -d 130.192.16.171 -j DROP", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-drop-keyword-hex", + args: []string{ + "-iptables-drop-keyword-hex", "|07 65 78 61 6d 70 6c 65 03 63 6f 6d|", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + `+ iptables -A JAFAR_OUTPUT -m string --algo kmp --hex-string "|07 65 78 61 6d 70 6c 65 03 63 6f 6d|" -j DROP`, + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-drop-keyword", + args: []string{ + "-iptables-drop-keyword", "ooni.org", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -A JAFAR_OUTPUT -m string --algo kmp --string ooni.org -j DROP", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-reset-ip", + args: []string{ + "-iptables-reset-ip", "130.192.16.171", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -A JAFAR_OUTPUT --proto tcp -d 130.192.16.171 -j REJECT --reject-with tcp-reset", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-reset-keyword-hex", + args: []string{ + "-iptables-reset-keyword-hex", "|6F 6F 6E 69|", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + `+ iptables -A JAFAR_OUTPUT -m string --proto tcp --algo kmp --hex-string "|6F 6F 6E 69|" -j REJECT --reject-with tcp-reset`, + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-reset-keyword", + args: []string{ + "-iptables-reset-keyword", "ooni.org", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -A JAFAR_OUTPUT -m string --proto tcp --algo kmp --string ooni.org -j REJECT --reject-with tcp-reset", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + var builder strings.Builder + input := append([]string{}, tc.args...) + input = append(input, "-dry-run") + + sigChan := make(chan os.Signal) + close(sigChan) // so mainWithArgs would not block + + t.Logf("executing with %+v", input) + mainWithArgs(&builder, sigChan, input...) + + output := builder.String() + lines := strings.Split(output, "\n") + if diff := cmp.Diff(tc.expect, lines); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/internal/engine/.gitignore b/internal/engine/.gitignore index 266f6d781f..1eef38ff89 100644 --- a/internal/engine/.gitignore +++ b/internal/engine/.gitignore @@ -1,16 +1,4 @@ -/*.jsonl -/.vscode -/apitool -/asn.mmdb -/ca-bundle.pem -/country.mmdb /example.org -/jafar -/jafar.exe -/miniooni -/miniooni.exe -/oohelper -/oohelperd /oonipsiphon/ /psiphon-config.json.age /psiphon-config.key diff --git a/internal/experiment/tor/tor_test.go b/internal/experiment/tor/tor_test.go index 9ce8290f81..2aa97b5fa1 100644 --- a/internal/experiment/tor/tor_test.go +++ b/internal/experiment/tor/tor_test.go @@ -815,10 +815,10 @@ func TestTestKeysFillToplevelKeysCoverMissingFields(t *testing.T) { failureString := "eof_error" tk := &TestKeys{ Targets: map[string]TargetResults{ - "foobar": {Failure: &failureString, TargetProtocol: "dir_port"}, - "baz": {TargetProtocol: "dir_port"}, - "jafar": {Failure: &failureString, TargetProtocol: "or_port_dirauth"}, - "jasmine": {TargetProtocol: "or_port_dirauth"}, + "foobar": {Failure: &failureString, TargetProtocol: "dir_port"}, + "baz": {TargetProtocol: "dir_port"}, + "ariel": {Failure: &failureString, TargetProtocol: "or_port_dirauth"}, + "sebastian": {TargetProtocol: "or_port_dirauth"}, }, } tk.fillToplevelKeys() diff --git a/internal/cmd/jafar/flagx/stringarray.go b/internal/flagx/stringarray.go similarity index 100% rename from internal/cmd/jafar/flagx/stringarray.go rename to internal/flagx/stringarray.go diff --git a/internal/cmd/jafar/flagx/stringarray_test.go b/internal/flagx/stringarray_test.go similarity index 88% rename from internal/cmd/jafar/flagx/stringarray_test.go rename to internal/flagx/stringarray_test.go index 256d1ed90d..4b63181375 100644 --- a/internal/cmd/jafar/flagx/stringarray_test.go +++ b/internal/flagx/stringarray_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/flagx" + "github.com/ooni/probe-cli/v3/internal/flagx" ) func TestStringArray(t *testing.T) { @@ -71,3 +71,7 @@ func TestStringArray(t *testing.T) { func assertFlagGetterStringArray(b flagx.StringArray) { func(in flag.Getter) {}(&b) } + +// Make sure the compiler does not complain about an unused function, otherwise next +// time I may end up removing it without noticing it's actually necessary. +var _ = assertFlagGetterStringArray diff --git a/internal/shellx/shellx.go b/internal/shellx/shellx.go index 8d09fbf62b..e47ad4c7fe 100644 --- a/internal/shellx/shellx.go +++ b/internal/shellx/shellx.go @@ -138,7 +138,7 @@ func cmd(config *Config, argv *Argv, envp *Envp) *execabs.Cmd { cmd.Env = append(cmd.Env, entry) } if config.Logger != nil { - cmdline := quotedCommandLineUnsafe(argv.P, argv.V...) + cmdline := QuotedCommandLineUnsafe(argv.P, argv.V...) config.Logger.Infof("+ %s", cmdline) } return cmd @@ -271,9 +271,9 @@ func OutputCommandLine(logger model.Logger, cmdline string) ([]byte, error) { // ErrNoCommandToExecute means that the command line is empty. var ErrNoCommandToExecute = errors.New("shellx: no command to execute") -// quotedCommandLineUnsafe returns a quoted command line. This function is unsafe +// QuotedCommandLineUnsafe returns a quoted command line. This function is unsafe // and SHOULD only be used to produce a nice output. -func quotedCommandLineUnsafe(command string, args ...string) string { +func QuotedCommandLineUnsafe(command string, args ...string) string { v := []string{} v = append(v, maybeQuoteArgUnsafe(command)) for _, a := range args { diff --git a/internal/tutorial/README.md b/internal/tutorial/README.md index 88a80f711b..eaa0e9861c 100644 --- a/internal/tutorial/README.md +++ b/internal/tutorial/README.md @@ -18,7 +18,9 @@ you should check how we do that in [internal/registry](../registry). - [Using the measurex package to write network experiments](measurex): this tutorial explains to you how to use the `measurex` library to write networking code that generates measurements using the OONI data format. You will learn -how to perform DNS, TCP, TLS, QUIC, HTTP, HTTPS, and HTTP3 measurements. +how to perform DNS, TCP, TLS, QUIC, HTTP, HTTPS, and HTTP3 measurements. (Note +that `measurex` is currently deprecated and we need to publish a new tutorial +about an improved approach to measuring, based on a DSL.) - [Low-level networking using netxlite](netxlite): this tutorial introduces you to the `netxlite` networking library. This is the underlying library diff --git a/internal/tutorial/experiment/torsf/chapter01/README.md b/internal/tutorial/experiment/torsf/chapter01/README.md index fe8f4b3821..1c61e55db6 100644 --- a/internal/tutorial/experiment/torsf/chapter01/README.md +++ b/internal/tutorial/experiment/torsf/chapter01/README.md @@ -243,7 +243,7 @@ we will just pretty-print the measurement on the `stdout`. You can now run this code as follows: ``` -$ go run ./experiment/torsf/chapter01 | jq +$ go run ./experiment/torsf/chapter01 | tail -n 1 | jq [snip] { "data_format_version": "", diff --git a/internal/tutorial/experiment/torsf/chapter03/README.md b/internal/tutorial/experiment/torsf/chapter03/README.md index 3ae6719ceb..c1aa8df3fe 100644 --- a/internal/tutorial/experiment/torsf/chapter03/README.md +++ b/internal/tutorial/experiment/torsf/chapter03/README.md @@ -128,7 +128,7 @@ func (m *Measurer) run(ctx context.Context, It's now time to run the new code we've written: ``` -$ go run ./experiment/torsf/chapter03 | jq +$ go run ./experiment/torsf/chapter03 | tail -n 1 | jq 2021/06/21 21:21:18 info [ 0.1%] torsf experiment is running 2021/06/21 21:21:19 info [ 0.2%] torsf experiment is running [...] diff --git a/internal/tutorial/experiment/torsf/chapter04/README.md b/internal/tutorial/experiment/torsf/chapter04/README.md index b90dd53bed..7e58d13992 100644 --- a/internal/tutorial/experiment/torsf/chapter04/README.md +++ b/internal/tutorial/experiment/torsf/chapter04/README.md @@ -163,7 +163,7 @@ so we just close the tunnel and record the bootstrap time. We can now run the code as follows to obtain: ``` -$ go run ./experiment/torsf/chapter04 | jq +$ go run ./experiment/torsf/chapter04 | tail -n 1 | jq [...] Jun 21 23:40:50.000 [notice] Bootstrapped 100% (done): Done 2021/06/21 23:40:50 info [100.0%] torsf experiment is finished diff --git a/internal/tutorial/measurex/README.md b/internal/tutorial/measurex/README.md index f89f9aa476..3f8e71e9b9 100644 --- a/internal/tutorial/measurex/README.md +++ b/internal/tutorial/measurex/README.md @@ -1,5 +1,8 @@ # Using the measurex package to write network experiments +**Note**: measurex is deprecated. We will remove this tutorial +once we have a tutorial for the new DSL-based measurement package. + This tutorial teaches you how to write OONI network experiments using the primitives in the `./internal/measurex` package. The name of this package means either "measure diff --git a/internal/tutorial/measurex/chapter04/README.md b/internal/tutorial/measurex/chapter04/README.md index 20504fb150..5f0664e417 100644 --- a/internal/tutorial/measurex/chapter04/README.md +++ b/internal/tutorial/measurex/chapter04/README.md @@ -221,14 +221,14 @@ To emulate the last two scenarios, if you're on Linux, a possibility is building Jafar with this command: ``` -go build -v ./internal/cmd/jafar +go build -v ./internal/cmd/tinyjafar ``` Then, for example, to provoke a connection reset you can run in a terminal: ``` -sudo ./jafar -iptables-reset-keyword dns.google +sudo ./tinyjafar -iptables-reset-keyword dns.google ``` and you can run this tutorial with `dns.google` as diff --git a/internal/tutorial/measurex/chapter04/main.go b/internal/tutorial/measurex/chapter04/main.go index bf9a2a21d6..25295eafbd 100644 --- a/internal/tutorial/measurex/chapter04/main.go +++ b/internal/tutorial/measurex/chapter04/main.go @@ -222,14 +222,14 @@ func main() { // possibility is building Jafar with this command: // // ``` -// go build -v ./internal/cmd/jafar +// go build -v ./internal/cmd/tinyjafar // ``` // // Then, for example, to provoke a connection reset you // can run in a terminal: // // ``` -// sudo ./jafar -iptables-reset-keyword dns.google +// sudo ./tinyjafar -iptables-reset-keyword dns.google // ``` // // and you can run this tutorial with `dns.google` as diff --git a/internal/tutorial/netxlite/chapter02/README.md b/internal/tutorial/netxlite/chapter02/README.md index 64a5fb5ebf..ef5d9eb2f1 100644 --- a/internal/tutorial/netxlite/chapter02/README.md +++ b/internal/tutorial/netxlite/chapter02/README.md @@ -216,11 +216,11 @@ named `ssl_invalid_hostname`). ### TLS handshake reset -If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) and then run: ```bash -sudo ./jafar -iptables-reset-keyword dns.google +sudo ./tinyjafar -iptables-reset-keyword dns.google ``` Then run in another terminal @@ -233,11 +233,11 @@ Then you can interrupt Jafar using ^C. ### TLS handshake timeout -If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) and then run: ```bash -sudo ./jafar -iptables-drop-keyword dns.google +sudo ./tinyjafar -iptables-drop-keyword dns.google ``` Then run in another terminal diff --git a/internal/tutorial/netxlite/chapter02/main.go b/internal/tutorial/netxlite/chapter02/main.go index 56ca4854e6..76b5dd3f73 100644 --- a/internal/tutorial/netxlite/chapter02/main.go +++ b/internal/tutorial/netxlite/chapter02/main.go @@ -217,11 +217,11 @@ func fatal(err error) { // // ### TLS handshake reset // -// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) // and then run: // // ```bash -// sudo ./jafar -iptables-reset-keyword dns.google +// sudo ./tinyjafar -iptables-reset-keyword dns.google // ``` // // Then run in another terminal @@ -234,11 +234,11 @@ func fatal(err error) { // // ### TLS handshake timeout // -// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) // and then run: // // ```bash -// sudo ./jafar -iptables-drop-keyword dns.google +// sudo ./tinyjafar -iptables-drop-keyword dns.google // ``` // // Then run in another terminal diff --git a/internal/tutorial/netxlite/chapter03/README.md b/internal/tutorial/netxlite/chapter03/README.md index 47f59d4b1d..cd4074be0a 100644 --- a/internal/tutorial/netxlite/chapter03/README.md +++ b/internal/tutorial/netxlite/chapter03/README.md @@ -158,11 +158,11 @@ named `ssl_invalid_hostname`). ### TLS handshake reset -If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) and then run: ```bash -sudo ./jafar -iptables-reset-keyword dns.google +sudo ./tinyjafar -iptables-reset-keyword dns.google ``` Then run in another terminal @@ -175,11 +175,11 @@ Then you can interrupt Jafar using ^C. ### TLS handshake timeout -If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) and then run: ```bash -sudo ./jafar -iptables-drop-keyword dns.google +sudo ./tinyjafar -iptables-drop-keyword dns.google ``` Then run in another terminal diff --git a/internal/tutorial/netxlite/chapter03/main.go b/internal/tutorial/netxlite/chapter03/main.go index 68f5d66d06..a4c38bacac 100644 --- a/internal/tutorial/netxlite/chapter03/main.go +++ b/internal/tutorial/netxlite/chapter03/main.go @@ -159,11 +159,11 @@ func fatal(err error) { // // ### TLS handshake reset // -// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) // and then run: // // ```bash -// sudo ./jafar -iptables-reset-keyword dns.google +// sudo ./tinyjafar -iptables-reset-keyword dns.google // ``` // // Then run in another terminal @@ -176,11 +176,11 @@ func fatal(err error) { // // ### TLS handshake timeout // -// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) // and then run: // // ```bash -// sudo ./jafar -iptables-drop-keyword dns.google +// sudo ./tinyjafar -iptables-drop-keyword dns.google // ``` // // Then run in another terminal diff --git a/script/testjafar.bash b/script/testjafar.bash deleted file mode 100755 index 0c415dcc11..0000000000 --- a/script/testjafar.bash +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - -# -# This script uses cURL to verify that Jafar is able to produce a -# bunch of censorship conditions. It should be noted that this script -# only works on Linux and will never work on other systems. -# - -set -e - -function execute() { - echo "+ $@" 1>&2 - "$@" -} - -function expectexitcode() { - local expect - local exitcode - expect=$1 - shift - set +e - "$@" - exitcode=$? - set -e - echo "expected exitcode $expect, found $exitcode" 1>&2 - if [ $exitcode != $expect ]; then - exit 1 - fi -} - -function runtest() { - echo "=== BEGIN $1 ===" - "$1" - echo "=== END $1 ===" -} - -function http_got_nothing() { - expectexitcode 52 execute ./jafar -iptables-hijack-http-to 127.0.0.1:7117 \ - -main-command 'curl -sm5 --connect-to ::example.com: http://ooni.io' -} - -function http_recv_error() { - expectexitcode 56 execute ./jafar -iptables-reset-keyword ooni \ - -main-command 'curl -sm5 --connect-to ::example.com: http://ooni.io' -} - -function http_operation_timedout() { - expectexitcode 28 execute ./jafar -iptables-drop-keyword ooni \ - -main-command 'curl -sm5 --connect-to ::example.com: http://ooni.io' -} - -function http_couldnt_connect() { - local ip - ip=$(host -tA example.com|cut -f4 -d' ') - expectexitcode 7 execute ./jafar -iptables-reset-ip $ip \ - -main-command 'curl -sm5 --connect-to ::example.com: http://ooni.io' -} - -function http_blockpage() { - outfile=$(mktemp) - chown nobody $outfile # curl runs as user nobody - expectexitcode 0 execute ./jafar -http-proxy-block ooni \ - -iptables-hijack-http-to 127.0.0.1:80 \ - -main-command "curl -so $outfile --connect-to ::example.com: http://ooni.io" - if ! grep -q '451 Unavailable For Legal Reasons' $outfile; then - echo "fatal: the blockpage does not contain the expected pattern" 1>&2 - exit 1 - fi -} - -function dns_injection() { - output=$(expectexitcode 0 execute ./jafar \ - -iptables-hijack-dns-to 127.0.0.1:53 \ - -dns-proxy-hijack ooni \ - -main-command 'dig +time=2 +short @example.com ooni.io') - if [ "$output" != "127.0.0.1" ]; then - echo "fatal: the resulting IP is not the expected one" 1>&2 - exit 1 - fi -} - -function dns_timeout() { - expectexitcode 9 execute ./jafar \ - -iptables-hijack-dns-to 127.0.0.1:53 \ - -dns-proxy-ignore ooni \ - -main-command 'dig +time=2 +short @example.com ooni.io' -} - -function dns_nxdomain() { - output=$(expectexitcode 0 execute ./jafar \ - -iptables-hijack-dns-to 127.0.0.1:53 \ - -dns-proxy-block ooni \ - -main-command 'dig +time=2 +short @example.com ooni.io') - if [ "$output" != "" ]; then - echo "fatal: expected no output here" 1>&2 - exit 1 - fi -} - -function sni_man_in_the_middle() { - expectexitcode 60 execute ./jafar -iptables-hijack-https-to 127.0.0.1:4114 \ - -main-command 'curl -sm5 --connect-to ::example.com: https://ooni.io' -} - -function sni_got_nothing() { - expectexitcode 52 execute ./jafar -iptables-hijack-https-to 127.0.0.1:4114 \ - -main-command 'curl -sm5 --cacert badproxy.pem --connect-to ::example.com: https://ooni.io' -} - -function sni_connect_error() { - expectexitcode 35 execute ./jafar -iptables-reset-keyword ooni \ - -main-command 'curl -sm5 --connect-to ::example.com: https://ooni.io' -} - -function main() { - runtest http_got_nothing - runtest http_recv_error - runtest http_operation_timedout - runtest http_couldnt_connect - runtest http_blockpage - runtest dns_injection - runtest dns_timeout - runtest dns_nxdomain - runtest sni_man_in_the_middle - runtest sni_got_nothing - runtest sni_connect_error -} - -main "$@"