diff --git a/.github/workflows/jafar.yml b/.github/workflows/jafar.yml deleted file mode 100644 index 546c5adeb9..0000000000 --- a/.github/workflows/jafar.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Checks whether the jafar tool is still WAI. -name: jafar -on: - push: - branches: - - "release/**" - - "fullbuild" - -jobs: - test: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - - name: Get GOVERSION content - id: goversion - run: echo "version=$(cat GOVERSION)" >> "$GITHUB_OUTPUT" - - - uses: magnetikonline/action-golang-cache@v4 - with: - go-version: "${{ steps.goversion.outputs.version }}" - cache-key-suffix: "-jafar-${{ steps.goversion.outputs.version }}" - - - run: go build -v ./internal/cmd/jafar - - - run: sudo ./script/testjafar.bash diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml deleted file mode 100644 index 9c98c07bc7..0000000000 --- a/.github/workflows/qa.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Runs quality assurance checks -name: "qa" -on: - push: - branches: - - "release/**" - - "fullbuild" - - "qabuild" - -jobs: - test_webconnectivity: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - run: ./QA/rundocker.bash "webconnectivity" diff --git a/.gitignore b/.gitignore index a011757c85..d632bd8b77 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ /citizenlab-test-lists /gardener /ghpublish.out.txt -/jafar /libooniengine.* /measurement.json /miniooni @@ -25,4 +24,5 @@ /ooniprobe /ooporthelper /probe-cli.cov +/tinyjafar /tmp-* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e2e41acae6..0652f4877a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,10 +88,9 @@ will understand you want to use the default pool) ## Code testing requirements Make sure all tests pass with `go test -race ./...` run from the -top-level directory of this repository. (Integration tests may be -flaky, so there may be some failures here and and there; we know -in particular that `./internal/cmd/jafar` is one of the usual -suspects and that it's not super pleasant to test it under Linux.) +top-level directory of this repository. (Running [netem]( +https://github.com/ooni/netem) based tests may not work as intended +with `-race` with macOS.) ## Writing a new OONI experiment diff --git a/QA/.dockerignore b/QA/.dockerignore deleted file mode 100644 index 72e8ffc0db..0000000000 --- a/QA/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/QA/.gitignore b/QA/.gitignore deleted file mode 100644 index 50dab8e9c0..0000000000 --- a/QA/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/Dockerfile -/GOPATH -/GOCACHE -/__pycache__ diff --git a/QA/README.md b/QA/README.md deleted file mode 100644 index 4600db93da..0000000000 --- a/QA/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Quality assurance scripts - -This directory contains quality assurance scripts that use Jafar to -ensure that OONI implementations behave. These scripts work with miniooni. - -## Run QA using a docker container - -Run test in a suitable Docker container using: - -```bash -./QA/rundocker.bash $nettest -``` - -Note that this will run a `--privileged` docker container. - -## Diagnosing issues - -The Python script that performs the QA runs a specific OONI test under -different failure conditions and stops at the first unexpected value found -in the resulting JSONL report. You can infer what went wrong by reading -the output of the `miniooni` command itself, which should be above the point -where the Python script stopped, as well as by inspecting the JSONL file on -disk. By convention such file is named `$nettest.jsonl` and only contains -the result of the last run of `$nettest`. diff --git a/QA/common.py b/QA/common.py deleted file mode 100644 index 656cb0027a..0000000000 --- a/QA/common.py +++ /dev/null @@ -1,68 +0,0 @@ -""" ./QA/common.py - common code for QA """ - -import contextlib -import json -import os -import shutil -import socket -import subprocess - - -def execute(args): - """ Execute a specified command """ - subprocess.run(args) - - -def execute_jafar_and_miniooni(ooni_exe, outfile, experiment, tag, args): - """ Executes jafar and miniooni. Returns the test keys. """ - tmpoutfile = "/tmp/{}".format(outfile) - with contextlib.suppress(FileNotFoundError): - os.remove(tmpoutfile) # just in case - execute( - [ - "./jafar", - "-main-command", - "./QA/minioonilike.py {} -n -o '{}' --home /tmp {}".format( - ooni_exe, tmpoutfile, experiment - ), - "-main-user", - "nobody", # should be present on Unix - "-tag", - tag, - ] - + args - ) - shutil.copy(tmpoutfile, outfile) - result = read_result(outfile) - assert isinstance(result, dict) - assert isinstance(result["test_keys"], dict) - return result["test_keys"] - - -def read_result(outfile): - """ Reads the result of an experiment """ - return json.load(open(outfile, "rb")) - - -def test_keys(result): - """ Returns just the test keys of a specific result """ - return result["test_keys"] - - -def check_maybe_binary_value(value): - """ Make sure a maybe binary value is correct """ - assert isinstance(value, str) or ( - isinstance(value, dict) - and value["format"] == "base64" - and isinstance(value["data"], str) - ) - - -def with_free_port(func): - """ This function executes |func| passing it a port number on localhost - which is bound but not listening for new connections """ - # See - with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - sock.bind(("127.0.0.1", 0)) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - func(sock.getsockname()[1]) diff --git a/QA/dockermain.sh b/QA/dockermain.sh deleted file mode 100755 index fab293da29..0000000000 --- a/QA/dockermain.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -set -euxo pipefail - -# required because the container is running as root -git config --global --add safe.directory /jafar - -# TODO(bassosimone): investigate why using CGO_ENABLED=1 is such -# that all DNS lookups return `dns_nxdomain_error` -export CGO_ENABLED=0 - -# TODO(bassosimone): because this script runs as root, it's not -# possible to save the caching directories in github actions but -# doing that would making re-executing these scripts faster. -export GOPATH=/jafar/QA/GOPATH -export GOCACHE=/jafar/QA/GOCACHE - -go build -v ./internal/cmd/miniooni - -go build -v ./internal/cmd/jafar - -sudo ./QA/$1.py ./miniooni diff --git a/QA/minioonilike.py b/QA/minioonilike.py deleted file mode 100755 index a4b9dda3c1..0000000000 --- a/QA/minioonilike.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 - -""" This script takes in input the name of the tool to run followed by - arguments and followed by the nettest name. The format recognized is - the same of miniooni. Depending on the tool that we want to run, we - reorder arguments so that they make sense for the tool. - - This is necessary because, albeit miniooni, MK, and OONI v2.x have - more or less the same arguments, there are some differences. We could - modify other tools to match miniooni, but this seems useless. """ - -import argparse -import os -import shlex -import sys - -sys.path.insert(0, ".") -import common - - -def file_must_exist(pathname): - """ Throws an exception if the given file does not actually exist. """ - if not os.path.isfile(pathname): - raise RuntimeError("missing {}: please run miniooni first".format(pathname)) - return pathname - - -def main(): - apa = argparse.ArgumentParser() - apa.add_argument("command", nargs=1, help="command to execute") - - # subset of arguments accepted by miniooni - apa.add_argument( - "-n", "--no-collector", action="count", help="don't submit measurement" - ) - apa.add_argument("-o", "--reportfile", help="specify report file to use") - apa.add_argument("-i", "--input", help="input for nettests taking an input") - apa.add_argument("--home", help="override home directory") - apa.add_argument("nettest", nargs=1, help="nettest to run") - out = apa.parse_args() - command, nettest = out.command[0], out.nettest[0] - - if "miniooni" not in command and "measurement_kit" not in command: - raise RuntimeError("unrecognized tool") - - args = [] - args.append(command) - if "miniooni" in command: - args.extend(["--yes"]) # make sure we have informed consent - if "measurement_kit" in command: - args.extend( - [ - "--ca-bundle-path", - file_must_exist("{}/.miniooni/assets/ca-bundle.pem".format(out.home)), - ] - ) - args.extend( - [ - "--geoip-country-path", - file_must_exist("{}/.miniooni/assets/country.mmdb".format(out.home)), - ] - ) - args.extend( - [ - "--geoip-asn-path", - file_must_exist("{}/.miniooni/assets/asn.mmdb".format(out.home)), - ] - ) - if out.home and "miniooni" in command: - args.extend(["--home", out.home]) # home applies to miniooni only - if out.input: - if "miniooni" in command: - args.extend(["-i", out.input]) # input is -i for miniooni - if out.no_collector: - args.append("-n") - if out.reportfile: - args.extend(["-o", out.reportfile]) - args.append(nettest) - if out.input and "measurement_kit" in command: - if nettest == "web_connectivity": - args.extend(["-u", out.input]) # MK's Web Connectivity uses -u for input - - sys.stderr.write("minioonilike.py: {}\n".format(shlex.join(args))) - common.execute(args) - - -if __name__ == "__main__": - main() diff --git a/QA/rundocker.bash b/QA/rundocker.bash deleted file mode 100755 index 77ec540b55..0000000000 --- a/QA/rundocker.bash +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -DOCKER=${DOCKER:-docker} - -GOVERSION=$(cat GOVERSION) - -cat > QA/Dockerfile << EOF -FROM golang:$GOVERSION-alpine -RUN apk add gcc go git musl-dev iptables tmux bind-tools curl sudo python3 -EOF - -$DOCKER build -t jafar-qa ./QA/ - -$DOCKER run --privileged -v$(pwd):/jafar -w/jafar jafar-qa ./QA/dockermain.sh "$@" diff --git a/QA/webconnectivity.py b/QA/webconnectivity.py deleted file mode 100755 index 3c8555cc07..0000000000 --- a/QA/webconnectivity.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 - - -""" ./QA/webconnectivity.py - main QA script for webconnectivity - - This script performs a bunch of webconnectivity tests under censored - network conditions and verifies that the measurement is consistent - with the expectations, by parsing the resulting JSONL. """ - -import socket -import sys -import time - -sys.path.insert(0, ".") -import common - - -def execute_jafar_and_return_validated_test_keys( - ooni_exe, outfile, experiment_args, tag, args -): - """Executes jafar and returns the validated parsed test keys, or throws - an AssertionError if the result is not valid.""" - tk = common.execute_jafar_and_miniooni( - ooni_exe, outfile, experiment_args, tag, args - ) - print("dns_experiment_failure", tk["dns_experiment_failure"], file=sys.stderr) - print("dns_consistency", tk["dns_consistency"], file=sys.stderr) - print("control_failure", tk["control_failure"], file=sys.stderr) - print("http_experiment_failure", tk["http_experiment_failure"], file=sys.stderr) - print("tk_body_length_match", tk["body_length_match"], file=sys.stderr) - print("body_proportion", tk["body_proportion"], file=sys.stderr) - print("status_code_match", tk["status_code_match"], file=sys.stderr) - print("headers_match", tk["headers_match"], file=sys.stderr) - print("title_match", tk["title_match"], file=sys.stderr) - print("blocking", tk["blocking"], file=sys.stderr) - print("accessible", tk["accessible"], file=sys.stderr) - print("x_status", tk["x_status"], file=sys.stderr) - return tk - - -def assert_status_flags_are(ooni_exe, tk, desired): - """Checks whether the status flags are what we expect them to - be when we're running miniooni. This check only makes sense - with miniooni b/c status flags are a miniooni extension.""" - if "miniooni" not in ooni_exe: - return - assert tk["x_status"] == desired - - -def webconnectivity_https_self_signed(ooni_exe, outfile): - """Test case where the certificate is self signed""" - args = [] - tk = execute_jafar_and_return_validated_test_keys( - ooni_exe, - outfile, - "-i https://self-signed.badssl.com/ web_connectivity", - "webconnectivity_https_self_signed", - args, - ) - assert tk["dns_experiment_failure"] == None - assert tk["dns_consistency"] == "consistent" - assert tk["control_failure"] == None - if "miniooni" in ooni_exe: - assert tk["http_experiment_failure"] == "ssl_unknown_authority" - else: - assert "certificate verify failed" in tk["http_experiment_failure"] - assert tk["body_length_match"] == None - assert tk["body_proportion"] == 0 - assert tk["status_code_match"] == None - assert tk["headers_match"] == None - assert tk["title_match"] == None - # The following strikes me as a measurement_kit bug. We are saying - # that all is good with a domain where actually we don't know why the - # control is failed and that is clearly not accessible according to - # our measurement of the domain (self signed certificate). - # - # See . - if "miniooni" in ooni_exe: - assert tk["blocking"] == None - assert tk["accessible"] == None - else: - assert tk["blocking"] == False - assert tk["accessible"] == True - assert_status_flags_are(ooni_exe, tk, 16) - - -def main(): - if len(sys.argv) != 2: - sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0]) - outfile = "webconnectivity.jsonl" - ooni_exe = sys.argv[1] - tests = [ - webconnectivity_https_self_signed, - ] - for test in tests: - test(ooni_exe, outfile) - time.sleep(7) - - -if __name__ == "__main__": - main() diff --git a/internal/cmd/jafar/.gitignore b/internal/cmd/jafar/.gitignore deleted file mode 100644 index 8e4647b852..0000000000 --- a/internal/cmd/jafar/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/badproxy.pem diff --git a/internal/cmd/jafar/README.md b/internal/cmd/jafar/README.md deleted file mode 100644 index 01081bdfc4..0000000000 --- a/internal/cmd/jafar/README.md +++ /dev/null @@ -1,257 +0,0 @@ -# Jafar - -> We stepped up the game of simulating censorship upgrading from the -> evil genius to the evil grand vizier. - -Jafar is a censorship simulation tool used for testing OONI. It builds on -any system but it really only works on Linux. - -## Building - -We use Go >= 1.20. Jafar also needs the C library headers, -iptables installed, and root permissions. - -With Linux Alpine edge, you can compile Jafar with: - -```bash -apk add go git musl-dev iptables -go build -v . -``` - -Otherwise, using Docker: - -```bash -docker build -t jafar-runner . -docker run -it --privileged -v`pwd`:/jafar -w/jafar jafar-runner -go build -v . -``` - -## Usage - -You need to run Jafar as root. You can get a complete list -of all flags using `./jafar -help`. Jafar is composed of modules. Each -module is controllable via flags. We describe modules below. - -### main - -The main module starts all the other modules. If you don't provide the -`-main-command ` flag, the code will run until interrupted. If -instead you use the `-main-command` flag, you can specify a command to -run inside the censored environment. In such case, the main module -will exit when the specified command terminates. Note that the main -module will propagate the child exit code, if the child fails. - -The command can also include arguments. Make sure you quote the arguments -such that your shell passes the whole string to the specified option, as -in `-main-command 'ls -lha'`. This will execute the `ls -lha` command line -inside the censored Jafar context. You can also combine that with quoting -and variables interpolation, e.g., `-main-command "echo '$USER is the -walrus'"`. The `$USER` variable will be expanded by your shell. Assuming -your user name is `paul`, then Jafar will lex the main command as `echo -"paul is the walrus"` and will execute it. - -Use the `-main-user ` flag to select the user to use for -running child commands. By default, we use the `nobody` user for this -purpose. We implement this feature using `sudo`, therefore you need -to make sure that `sudo` is installed. - -### iptables - -The iptables module is only available on Linux. It exports these flags: - -```bash - -iptables-drop-ip value - Drop traffic to the specified IP address - -iptables-drop-keyword-hex value - Drop traffic containing the specified hex keyword - -iptables-drop-keyword value - Drop traffic containing the specified keyword - -iptables-hijack-dns-to string - Hijack all DNS UDP traffic to the specified endpoint - -iptables-hijack-https-to string - Hijack all HTTPS traffic to the specified endpoint - -iptables-hijack-http-to string - Hijack all HTTP traffic to the specified endpoint - -iptables-reset-ip value - Reset TCP/IP traffic to the specified IP address - -iptables-reset-keyword-hex value - Reset TCP/IP traffic containing the specified hex keyword - -iptables-reset-keyword value - Reset TCP/IP traffic containing the specified keyword -``` - -The difference between `drop` and `reset` is that in the former case -a packet is dropped, in the latter case a RST is sent. - -The difference between `ip` and `keyword` flags is that the former -match an outgoing IP, the latter uses DPI. - -The `drop` and `reset` rules allow you to simulate, respectively, when -operations timeout and when a connection cannot be established (with -`reset` and `ip`) or is reset after a keyword is seen (with `keyword`). - -Hijacking DNS traffic is useful, for example, to redirect all DNS UDP -traffic from the box to the `dns-proxy` module. - -Hijacking HTTP and HTTPS traffic actually hijacks based on ports rather -than on DPI. As a known bug, when hijacking HTTP or HTTPS traffic, we -do not hijack traffic owned by root. This is because Jafar runs as root -and therefore its traffic must not match the hijack rule. - -When matching keywords, the simplest option is to use ASCII strings as -in `-iptables-drop-keyword ooni`. However, you can also specify a sequence -of hex bytes, as in `-iptables-drop-keyword-hex |6f 6f 6e 69|`. - -Note that with `-iptables-drop-keyword`, DNS queries containing such -keyword will fail returning `EPERM`. For a more realistic approach to -dropping specific DNS packets, combine DNS traffic hijacking with -`-dns-proxy-ignore`, to "drop" packets at the DNS proxy. - -### dns-proxy (aka resolver) - -The DNS proxy or resolver allows to manipulate DNS. Unless you use DNS -hijacking, you will need to configure your application explicitly to use -the proxy with application specific command line flags. - -```bash - -dns-proxy-address string - Address where the DNS proxy should listen (default "127.0.0.1:53") - -dns-proxy-block value - Register keyword triggering NXDOMAIN censorship - -dns-proxy-hijack value - Register keyword triggering redirection to 127.0.0.1 - -dns-proxy-ignore value - Register keyword causing the proxy to ignore the query -``` - -The `-dns-proxy-address` flag controls the endpoint where the proxy is -listening. - -The `-dns-proxy-block` tells the resolver that every incoming request whose -query contains the specifed string shall receive an `NXDOMAIN` reply. - -The `-dns-proxy-hijack` is similar but instead lies and returns to the -client that the requested domain is at `127.0.0.1`. This is an opportunity -to redirect traffic to the HTTP and TLS proxies. - -The `-dns-proxy-ignore` is similar but instead just ignores the query. - -### http-proxy - -The HTTP proxy is an HTTP proxy that may refuse to forward some -specific requests. It's controlled by these flags: - -```bash - -http-proxy-address string - Address where the HTTP proxy should listen (default "127.0.0.1:80") - -http-proxy-block value - Register keyword triggering HTTP 451 censorship -``` - -The `-http-proxy-address` flag has the same semantics it has for the DNS -proxy. - -The `-http-proxy-block` flag tells the proxy that it should return a `451` -response for every request whose `Host` contains the specified string. - -### tls-proxy - -TLS proxy is a TCP proxy that routes traffic to specific servers depending -on their SNI value. It is controlled by the following flags: - -```bash - -tls-proxy-address string - Address where the TCP+TLS proxy should listen (default "127.0.0.1:443") - -tls-proxy-block value - Register SNI header keyword triggering TLS censorship - -tls-proxy-outbound-port - Define the outbound port requests are proxied to (default "443" for HTTPS) -``` - -The `-tls-proxy-address` flags has the same semantics it has for the DNS -proxy. - -The `-tls-proxy-block` specifies which string or strings should cause the -proxy to return an internal-erorr alert when the incoming ClientHello's SNI -contains one of the strings provided with this option. - -### bad-proxy - -```bash - -bad-proxy-address string - Address where to listen for TCP connections (default "127.0.0.1:7117") - -bad-proxy-address-tls string - Address where to listen for TLS connections (default "127.0.0.1:4114") - -bad-proxy-tls-output-ca string - File where to write the CA used by the bad proxy (default "badproxy.pem") -``` - -The bad proxy is a proxy that reads some bytes from any incoming connection -and then closes the connection without replying anything. This simulates a -proxy that is not working properly, hence the name of the module. - -When connecting using TLS, the above behaviour happens after the handshake. - -We write the CA on the file specified using `-bad-proxy-tls-output-ca` such that -tools like curl(1) can use such CA to avoid TLS handshake errors. The code will -generate on the fly a certificate for the provided SNI. Not providing any SNI in -the client Hello message will cause the TLS handshake to fail. - -### uncensored - -```bash - -uncensored-resolver-doh string - URL of an hopefully uncensored DoH resolver (default "https://1.1.1.1/dns-query") -``` - -The HTTP, DNS, and TLS proxies need to resolve domain names. If you setup DNS -censorship, they may be affected as well. To avoid this issue, we use a different -resolver for them, which by default is the one shown above. You can change such -default by using the `-uncensored-resolver-doh` command line flag. The input -URL is an HTTPS URL pointing to a DoH server. Here are some examples: - -* `https://dns.google/dns-query` -* `https://dns.quad9.net/dns-query` - -So, for example, if you are using Jafar to censor `1.1.1.1:443`, then you -most likely want to use `-uncensored-resolver-doh`. - -## Examples - -Block `play.google.com` with RST injection, force DNS traffic to use the our -DNS proxy, and force it to censor `play.google.com` with `NXDOMAIN`. - -```bash -# ./jafar -iptables-reset-keyword play.google.com \ - -iptables-hijack-dns-to 127.0.0.1:5353 \ - -dns-proxy-address 127.0.0.1:5353 \ - -dns-proxy-block play.google.com -``` - -Force all traffic through the HTTP and TLS proxy and use them to censor -`play.google.com` using HTTP 451 and responding with TLS alerts: - -```bash -# ./jafar -iptables-hijack-dns-to 127.0.0.1:5353 \ - -dns-proxy-address 127.0.0.1:5353 \ - -dns-proxy-hijack play.google.com \ - -http-proxy-block play.google.com \ - -tls-proxy-block play.google.com -``` - -Run `ping` in a censored environment: - -```bash -# ./jafar -iptables-drop-ip 8.8.8.8 -main-command 'ping -c3 8.8.8.8' -``` - -Run `curl` in a censored environment where it cannot connect to -`play.google.com` using `https`: - -```bash -# ./jafar -iptables-hijack-https-to 127.0.0.1:443 \ - -tls-proxy-block play.google.com \ - -main-command 'curl -Lv http://play.google.com' -``` - -For more usage examples, see `../../script/testjafar.bash`. diff --git a/internal/cmd/jafar/badproxy/badproxy.go b/internal/cmd/jafar/badproxy/badproxy.go deleted file mode 100644 index 3690bb63e8..0000000000 --- a/internal/cmd/jafar/badproxy/badproxy.go +++ /dev/null @@ -1,114 +0,0 @@ -// Package badproxy implements misbehaving proxies. We have a single -// CensoringProxy that exports two misbehaving endpoints. Each endpoint -// implements a different proxy-censorsing technique. The first one -// reads some bytes from the connection then closes the connection. The -// other instead replies with a self signed x509 certificate. -package badproxy - -import ( - "context" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "io" - "net" - "strings" - "time" - - "github.com/google/martian/v3/mitm" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// CensoringProxy is a proxy that does not behave correctly. -type CensoringProxy struct { - mitmNewAuthority func( - name string, organization string, - validity time.Duration, - ) (*x509.Certificate, *rsa.PrivateKey, error) - - mitmNewConfig func( - ca *x509.Certificate, privateKey interface{}, - ) (*mitm.Config, error) - - tlsListen func( - network string, laddr string, config *tls.Config, - ) (net.Listener, error) -} - -// NewCensoringProxy creates a new instance of a misbehaving proxy. -func NewCensoringProxy() *CensoringProxy { - return &CensoringProxy{ - mitmNewAuthority: mitm.NewAuthority, - mitmNewConfig: mitm.NewConfig, - tlsListen: tls.Listen, - } -} - -func (p *CensoringProxy) serve(conn net.Conn) { - deadline := time.Now().Add(250 * time.Millisecond) - conn.SetDeadline(deadline) - // To simulate the case where the proxy isn't willing to forward our - // traffic, we close the connection (1) right after the handshake for - // TLS connections and (2) reasonably after we've received the HTTP - // request for cleartext connections. This may break in several cases - // but is good enough approximation of these bad proxies for now. - if tlsconn, ok := conn.(*tls.Conn); ok { - tlsconn.Handshake() - } else { - const maxread = 1 << 17 - reader := io.LimitReader(conn, maxread) - netxlite.ReadAllContext(context.Background(), reader) - } - conn.Close() -} - -func (p *CensoringProxy) run(listener net.Listener) { - for { - conn, err := listener.Accept() - if err != nil && strings.Contains( - err.Error(), "use of closed network connection") { - return - } - if err == nil { - // It's difficult to make accept fail, so restructure - // the code such that we enter into the happy path - go p.serve(conn) - } - } -} - -// Start starts the misbehaving proxy for TCP. This endpoint will read some -// bytes from the request and then close the connection. This behaviour is -// implemented by a bunch of censoring proxy around the world. Usually such -// proxies only close the connection with offending SNIs/Host headers. -func (p *CensoringProxy) Start(address string) (net.Listener, error) { - listener, err := net.Listen("tcp", address) - if err != nil { - return nil, err - } - go p.run(listener) - return listener, nil -} - -// StartTLS starts the misbehaving proxy for TLS. This endpoint will return -// to the client a self signed certificate. Thus, it models the case where a -// MITM forces users to accept a rogue certificate. After sending such a -// certificate, this proxy will close the TCP connection. -func (p *CensoringProxy) StartTLS(address string) (net.Listener, *x509.Certificate, error) { - cert, privkey, err := p.mitmNewAuthority( - "jafar", "OONI", 24*time.Hour, - ) - if err != nil { - return nil, nil, err - } - config, err := p.mitmNewConfig(cert, privkey) - if err != nil { - return nil, nil, err - } - listener, err := p.tlsListen("tcp", address, config.TLS()) - if err != nil { - return nil, nil, err - } - go p.run(listener) - return listener, cert, nil -} diff --git a/internal/cmd/jafar/badproxy/badproxy_test.go b/internal/cmd/jafar/badproxy/badproxy_test.go deleted file mode 100644 index 2b0769ba64..0000000000 --- a/internal/cmd/jafar/badproxy/badproxy_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package badproxy - -import ( - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "errors" - "net" - "testing" - "time" - - "github.com/google/martian/v3/mitm" -) - -func TestCleartext(t *testing.T) { - listener := newproxy(t) - checkdial(t, listener.Addr().String(), nil, net.Dial) - killproxy(t, listener) -} - -func TestTLS(t *testing.T) { - listener := newproxytls(t) - checkdial(t, listener.Addr().String(), nil, - func(network, address string) (net.Conn, error) { - conn, err := tls.Dial(network, address, &tls.Config{ - InsecureSkipVerify: true, - ServerName: "antani.local", - }) - if err != nil { - return nil, err - } - if err = conn.Handshake(); err != nil { - conn.Close() - return nil, err - } - return conn, nil - }) - killproxy(t, listener) -} - -func TestListenError(t *testing.T) { - proxy := NewCensoringProxy() - listener, err := proxy.Start("8.8.8.8:80") - if err == nil { - t.Fatal("expected an error here") - } - if listener != nil { - t.Fatal("expected nil listener here") - } -} - -func TestStartTLS(t *testing.T) { - expected := errors.New("mocked error") - - t.Run("when we cannot create a new authority", func(t *testing.T) { - proxy := NewCensoringProxy() - proxy.mitmNewAuthority = func( - name string, organization string, - validity time.Duration, - ) (*x509.Certificate, *rsa.PrivateKey, error) { - return nil, nil, expected - } - cert, privkey, err := proxy.StartTLS("127.0.0.1:0") - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } - if cert != nil { - t.Fatal("expected nil cert") - } - if privkey != nil { - t.Fatal("expected nil privkey") - } - }) - - t.Run("when we cannot create a new config", func(t *testing.T) { - proxy := NewCensoringProxy() - proxy.mitmNewConfig = func( - ca *x509.Certificate, privateKey interface{}, - ) (*mitm.Config, error) { - return nil, expected - } - cert, privkey, err := proxy.StartTLS("127.0.0.1:0") - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } - if cert != nil { - t.Fatal("expected nil cert") - } - if privkey != nil { - t.Fatal("expected nil privkey") - } - }) - - t.Run("when we cannot listen", func(t *testing.T) { - proxy := NewCensoringProxy() - proxy.tlsListen = func( - network string, laddr string, config *tls.Config, - ) (net.Listener, error) { - return nil, expected - } - cert, privkey, err := proxy.StartTLS("127.0.0.1:0") - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } - if cert != nil { - t.Fatal("expected nil cert") - } - if privkey != nil { - t.Fatal("expected nil privkey") - } - }) -} - -func newproxy(t *testing.T) net.Listener { - proxy := NewCensoringProxy() - listener, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return listener -} - -func newproxytls(t *testing.T) net.Listener { - proxy := NewCensoringProxy() - listener, _, err := proxy.StartTLS("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return listener -} - -func killproxy(t *testing.T, listener net.Listener) { - err := listener.Close() - if err != nil { - t.Fatal(err) - } -} - -func checkdial( - t *testing.T, proxyAddr string, expectErr error, - dial func(network, address string) (net.Conn, error), -) { - conn, err := dial("tcp", proxyAddr) - if err != expectErr { - t.Fatal("not the result we expected") - } - if conn == nil && expectErr == nil { - t.Fatal("expected actionable conn") - } - if conn != nil && expectErr != nil { - t.Fatal("expected nil conn") - } - if conn != nil { - conn.Write([]byte("123454321")) - conn.Close() - } -} diff --git a/internal/cmd/jafar/httpproxy/httpproxy.go b/internal/cmd/jafar/httpproxy/httpproxy.go deleted file mode 100644 index eac502510a..0000000000 --- a/internal/cmd/jafar/httpproxy/httpproxy.go +++ /dev/null @@ -1,78 +0,0 @@ -// Package httpproxy contains a censoring HTTP proxy. This proxy will -// vet all the traffic and reply with 451 responses for a configurable -// set of offending Host headers in incoming requests. -package httpproxy - -import ( - "net" - "net/http" - "net/http/httputil" - "net/url" - "strings" -) - -const product = "jafar/0.1.0" - -// CensoringProxy is a censoring HTTP proxy -type CensoringProxy struct { - keywords []string - transport http.RoundTripper -} - -// NewCensoringProxy creates a new CensoringProxy instance using -// the specified list of keywords to censor. keywords is the list -// of keywords that trigger censorship if any of them appears in -// the Host header of a request. dnsNetwork and dnsAddress are -// settings to configure the upstream, non censored DNS. -func NewCensoringProxy( - keywords []string, uncensored http.RoundTripper, -) *CensoringProxy { - return &CensoringProxy{keywords: keywords, transport: uncensored} -} - -var blockpage = []byte(` - 451 Unavailable For Legal Reasons - -

451 Unavailable For Legal Reasons

-

This content is not available in your jurisdiction.

- -`) - -// ServeHTTP serves HTTP requests -func (p *CensoringProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Implementation note: use Via header to detect in a loose way - // requests originated by us and directed to us - if r.Header.Get("Via") != "" || r.Host == "" { - w.WriteHeader(http.StatusBadRequest) - return - } - for _, pattern := range p.keywords { - if strings.Contains(r.Host, pattern) { - w.WriteHeader(http.StatusUnavailableForLegalReasons) - w.Write(blockpage) - return - } - } - r.Header.Add("Via", product) // see above - proxy := httputil.NewSingleHostReverseProxy(&url.URL{ - Host: r.Host, - Scheme: "http", - }) - proxy.ModifyResponse = func(resp *http.Response) error { - resp.Header.Add("Via", product) // see above - return nil - } - proxy.Transport = p.transport - proxy.ServeHTTP(w, r) -} - -// Start starts the censoring proxy. -func (p *CensoringProxy) Start(address string) (*http.Server, net.Addr, error) { - server := &http.Server{Handler: p} - listener, err := net.Listen("tcp", address) - if err != nil { - return nil, nil, err - } - go server.Serve(listener) - return server, listener.Addr(), nil -} diff --git a/internal/cmd/jafar/httpproxy/httpproxy_test.go b/internal/cmd/jafar/httpproxy/httpproxy_test.go deleted file mode 100644 index 399f78b43b..0000000000 --- a/internal/cmd/jafar/httpproxy/httpproxy_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package httpproxy - -import ( - "bytes" - "context" - "net" - "net/http" - "testing" - - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestPass(t *testing.T) { - server, addr := newproxy(t, "ooni.io") - // We're filtering ooni.io, so we expect example.com to pass - // through the proxy with 200 and we also expect to see the - // Via header in the responses we receive, of course. - checkrequest(t, addr.String(), "example.com", 200, true) - killproxy(t, server) -} - -func TestBlock(t *testing.T) { - server, addr := newproxy(t, "ooni.io") - // Here we're filtering any domain containing ooni.io, so we - // expect the proxy to send 451 without actually proxing, thus - // there should not be any Via header in the output. - checkrequest(t, addr.String(), "api.ooni.io", 451, false) - killproxy(t, server) -} - -func TestLoop(t *testing.T) { - server, addr := newproxy(t, "ooni.io") - // Here we're forcing the proxy to connect to itself. It does - // does that and recognizes itself because of the Via header - // being set in the request generated by the connection to itself, - // which should cause a 400. The response should have the Via - // header set because the 400 is received by the connection that - // this code has made to the proxy. - checkrequest(t, addr.String(), addr.String(), 400, true) - killproxy(t, server) -} - -func TestListenError(t *testing.T) { - proxy := NewCensoringProxy([]string{""}, uncensored.NewClient("https://1.1.1.1/dns-query")) - server, addr, err := proxy.Start("8.8.8.8:80") - if err == nil { - t.Fatal("expected an error here") - } - if server != nil { - t.Fatal("expected nil server here") - } - if addr != nil { - t.Fatal("expected nil addr here") - } -} - -func newproxy(t *testing.T, blocked string) (*http.Server, net.Addr) { - proxy := NewCensoringProxy([]string{blocked}, uncensored.NewClient("https://1.1.1.1/dns-query")) - server, addr, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return server, addr -} - -func killproxy(t *testing.T, server *http.Server) { - err := server.Shutdown(context.Background()) - if err != nil { - t.Fatal(err) - } -} - -func checkrequest( - t *testing.T, proxyAddr, host string, - expectStatus int, expectVia bool, -) { - req, err := http.NewRequest("GET", "http://"+proxyAddr, nil) - if err != nil { - t.Fatal(err) - } - req.Host = host - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != expectStatus { - t.Fatal("unexpected value of status code") - } - t.Log(resp) - values := resp.Header["Via"] - var foundProduct bool - for _, value := range values { - if value == product { - foundProduct = true - } - } - if foundProduct && !expectVia { - t.Fatal("unexpectedly found Via header") - } - if !foundProduct && expectVia { - t.Fatal("Via header not found") - } - proxiedData, err := netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - if expectStatus == 200 { - checkbody(t, proxiedData, host) - } -} - -func checkbody(t *testing.T, proxiedData []byte, host string) { - resp, err := http.Get("http://" + host) - if err != nil { - t.Fatal(err) - } - if resp.StatusCode != 200 { - t.Fatal("unexpected status code") - } - defer resp.Body.Close() - data, err := netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - if bytes.Equal(data, proxiedData) == false { - t.Fatal("body mismatch") - } -} diff --git a/internal/cmd/jafar/iptables/iptables.go b/internal/cmd/jafar/iptables/iptables.go deleted file mode 100644 index 580e8bf1df..0000000000 --- a/internal/cmd/jafar/iptables/iptables.go +++ /dev/null @@ -1,98 +0,0 @@ -// Package iptables contains code for managing firewall rules. This package -// really only works reliably on Linux. In all other systems the functionality -// in here is just a set of stubs returning errors. -package iptables - -import ( - "github.com/ooni/probe-cli/v3/internal/runtimex" -) - -type shell interface { - createChains() error - dropIfDestinationEquals(ip string) error - rstIfDestinationEqualsAndIsTCP(ip string) error - dropIfContainsKeywordHex(keyword string) error - dropIfContainsKeyword(keyword string) error - rstIfContainsKeywordHexAndIsTCP(keyword string) error - rstIfContainsKeywordAndIsTCP(keyword string) error - hijackDNS(address string) error - hijackHTTPS(address string) error - hijackHTTP(address string) error - waive() error -} - -// CensoringPolicy implements a censoring policy. -type CensoringPolicy struct { - DropIPs []string // drop IP traffic to these IPs - DropKeywordsHex []string // drop IP packets with these hex keywords - DropKeywords []string // drop IP packets with these keywords - HijackDNSAddress string // where to hijack DNS to - HijackHTTPSAddress string // where to hijack HTTPS to - HijackHTTPAddress string // where to hijack HTTP to - ResetIPs []string // RST TCP/IP traffic to these IPs - ResetKeywordsHex []string // RST TCP/IP flows with these hex keywords - ResetKeywords []string // RST TCP/IP flows with these keywords - sh shell -} - -// NewCensoringPolicy returns a new censoring policy. -func NewCensoringPolicy() *CensoringPolicy { - return &CensoringPolicy{ - sh: newShell(), - } -} - -// Apply applies the censorship policy -func (c *CensoringPolicy) Apply() (err error) { - defer func() { - if recover() != nil { - // JUST KNOW WE'VE BEEN HERE - } - }() - err = c.sh.createChains() - runtimex.PanicOnError(err, "c.sh.createChains failed") - // Implementation note: we want the RST rules to be first such - // that we end up enforcing them before the drop rules. - for _, keyword := range c.ResetKeywordsHex { - err = c.sh.rstIfContainsKeywordHexAndIsTCP(keyword) - runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordHexAndIsTCP failed") - } - for _, keyword := range c.ResetKeywords { - err = c.sh.rstIfContainsKeywordAndIsTCP(keyword) - runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordAndIsTCP failed") - } - for _, ip := range c.ResetIPs { - err = c.sh.rstIfDestinationEqualsAndIsTCP(ip) - runtimex.PanicOnError(err, "c.sh.rstIfDestinationEqualsAndIsTCP failed") - } - for _, keyword := range c.DropKeywordsHex { - err = c.sh.dropIfContainsKeywordHex(keyword) - runtimex.PanicOnError(err, "c.sh.dropIfContainsKeywordHex failed") - } - for _, keyword := range c.DropKeywords { - err = c.sh.dropIfContainsKeyword(keyword) - runtimex.PanicOnError(err, "c.sh.dropIfContainsKeyword failed") - } - for _, ip := range c.DropIPs { - err = c.sh.dropIfDestinationEquals(ip) - runtimex.PanicOnError(err, "c.sh.dropIfDestinationEquals failed") - } - if c.HijackDNSAddress != "" { - err = c.sh.hijackDNS(c.HijackDNSAddress) - runtimex.PanicOnError(err, "c.sh.hijackDNS failed") - } - if c.HijackHTTPSAddress != "" { - err = c.sh.hijackHTTPS(c.HijackHTTPSAddress) - runtimex.PanicOnError(err, "c.sh.hijackHTTPS failed") - } - if c.HijackHTTPAddress != "" { - err = c.sh.hijackHTTP(c.HijackHTTPAddress) - runtimex.PanicOnError(err, "c.sh.hijackHTTP failed") - } - return -} - -// Waive removes any censorship policy -func (c *CensoringPolicy) Waive() error { - return c.sh.waive() -} diff --git a/internal/cmd/jafar/iptables/iptables_integration_test.go b/internal/cmd/jafar/iptables/iptables_integration_test.go deleted file mode 100644 index 50f17228d9..0000000000 --- a/internal/cmd/jafar/iptables/iptables_integration_test.go +++ /dev/null @@ -1,348 +0,0 @@ -package iptables - -import ( - "context" - "errors" - "net" - "net/http" - "net/http/httptest" - "net/url" - "runtime" - "strings" - "testing" - "time" - - "golang.org/x/sys/execabs" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/resolver" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" - "github.com/ooni/probe-cli/v3/internal/shellx" -) - -func init() { - log.SetLevel(log.ErrorLevel) -} - -func newCensoringPolicy() *CensoringPolicy { - policy := NewCensoringPolicy() - policy.Waive() // start over to allow for repeated tests on failure - return policy -} - -func TestCannotApplyPolicy(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.DropIPs = []string{"antani"} - if err := policy.Apply(); err == nil { - t.Fatal("expected an error here") - } -} - -func TestCreateChainsError(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - // you should not be able to apply the policy when there is - // already a policy, you need to waive it first - if err := policy.Apply(); err == nil { - t.Fatal("expected an error here") - } -} - -func TestDropIP(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.DropIPs = []string{"1.1.1.1"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", "1.1.1.1:853") - if err == nil { - t.Fatalf("expected an error here") - } - if err.Error() != "dial tcp 1.1.1.1:853: i/o timeout" { - t.Fatal("unexpected error occurred") - } - if conn != nil { - t.Fatal("expected nil connection here") - } -} - -func TestDropKeyword(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.DropKeywords = []string{"ooni.io"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - req, err := http.NewRequest("GET", "http://www.ooni.io", nil) - if err != nil { - t.Fatal(err) - } - resp, err := http.DefaultClient.Do(req.WithContext(ctx)) - if err == nil { - t.Fatal("expected an error here") - } - if !strings.HasSuffix(err.Error(), "context deadline exceeded") { - t.Fatal("unexpected error occurred") - } - if resp != nil { - t.Fatal("expected nil response here") - } -} - -func TestDropKeywordHex(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.DropKeywordsHex = []string{"|6f 6f 6e 69|"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - reso := &net.Resolver{ - PreferGo: true, - } - addrs, err := reso.LookupHost(ctx, "www.ooni.io") - if err == nil { - t.Fatal("expected an error here") - } - // the error we see with GitHub Actions is different from the error - // we see when testing locally on Fedora - if !strings.HasSuffix(err.Error(), "operation not permitted") && - !strings.HasSuffix(err.Error(), "Temporary failure in name resolution") && - !strings.HasSuffix(err.Error(), "no such host") { - t.Fatalf("unexpected error occurred: %+v", err) - } - if addrs != nil { - t.Fatal("expected nil response here") - } -} - -func TestResetIP(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.ResetIPs = []string{"1.1.1.1"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - conn, err := (&net.Dialer{}).Dial("tcp", "1.1.1.1:853") - if err == nil { - t.Fatalf("expected an error here") - } - if err.Error() != "dial tcp 1.1.1.1:853: connect: connection refused" { - t.Fatal("unexpected error occurred") - } - if conn != nil { - t.Fatal("expected nil connection here") - } -} - -func TestResetKeyword(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.ResetKeywords = []string{"ooni.io"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - resp, err := http.Get("http://www.ooni.io") - if err == nil { - t.Fatal("expected an error here") - } - if strings.Contains(err.Error(), "read: connection reset by peer") == false { - t.Fatal("unexpected error occurred") - } - if resp != nil { - t.Fatal("expected nil response here") - } -} - -func TestResetKeywordHex(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - policy := newCensoringPolicy() - defer policy.Waive() - policy.ResetKeywordsHex = []string{"|6f 6f 6e 69|"} - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - resp, err := http.Get("http://www.ooni.io") - if err == nil { - t.Fatal("expected an error here") - } - if strings.Contains(err.Error(), "read: connection reset by peer") == false { - t.Fatal("unexpected error occurred") - } - if resp != nil { - t.Fatal("expected nil response here") - } -} - -func TestHijackDNS(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - resolver := resolver.NewCensoringResolver( - []string{"ooni.io"}, nil, nil, - uncensored.NewClient("https://1.1.1.1/dns-query"), - ) - server, err := resolver.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer server.Shutdown() - policy := newCensoringPolicy() - defer policy.Waive() - policy.HijackDNSAddress = server.PacketConn.LocalAddr().String() - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - reso := &net.Resolver{ - PreferGo: true, - } - addrs, err := reso.LookupHost(context.Background(), "www.ooni.io") - if err == nil { - t.Fatal("expected an error here") - } - if strings.Contains(err.Error(), "no such host") == false { - t.Fatal("unexpected error occurred") - } - if addrs != nil { - t.Fatal("expected nil addrs here") - } -} - -func TestHijackHTTP(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - // Implementation note: this test is complicated by the fact - // that we are running as root and so we're whitelisted. - server := httptest.NewServer(http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(451) - }), - ) - defer server.Close() - policy := newCensoringPolicy() - defer policy.Waive() - pu, err := url.Parse(server.URL) - if err != nil { - t.Fatal(err) - } - policy.HijackHTTPAddress = pu.Host - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - err = shellx.Run(log.Log, "sudo", "-u", "nobody", "--", - "curl", "-sf", "http://example.com") - if err == nil { - t.Fatal("expected an error here") - } - var exitErr *execabs.ExitError - if !errors.As(err, &exitErr) { - t.Fatal("not the error type we expected") - } - if exitErr.ExitCode() != 22 { - t.Fatal("not the exit code we expected") - } -} - -func TestHijackHTTPS(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("not implemented on this platform") - } - if testing.Short() { - t.Skip("skip test in short mode") - } - // Implementation note: this test is complicated by the fact - // that we are running as root and so we're whitelisted. - server := httptest.NewTLSServer(http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(451) - }), - ) - defer server.Close() - policy := newCensoringPolicy() - defer policy.Waive() - pu, err := url.Parse(server.URL) - if err != nil { - t.Fatal(err) - } - policy.HijackHTTPSAddress = pu.Host - if err := policy.Apply(); err != nil { - t.Fatal(err) - } - err = shellx.Run(log.Log, "sudo", "-u", "nobody", "--", - "curl", "-sf", "https://example.com") - if err == nil { - t.Fatal("expected an error here") - } - t.Log(err) - var exitErr *execabs.ExitError - if !errors.As(err, &exitErr) { - t.Fatal("not the error type we expected") - } - if exitErr.ExitCode() != 60 { - t.Fatal("not the exit code we expected") - } -} diff --git a/internal/cmd/jafar/iptables/iptables_linux.go b/internal/cmd/jafar/iptables/iptables_linux.go deleted file mode 100644 index 09d4c4dd52..0000000000 --- a/internal/cmd/jafar/iptables/iptables_linux.go +++ /dev/null @@ -1,119 +0,0 @@ -//go:build linux - -package iptables - -import ( - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "github.com/ooni/probe-cli/v3/internal/shellx" -) - -type linuxShell struct{} - -func (s *linuxShell) createChains() (err error) { - defer func() { - if recover() != nil { - // JUST KNOW WE'VE BEEN HERE - } - }() - err = shellx.Run(log.Log, "sudo", "iptables", "-N", "JAFAR_INPUT") - runtimex.PanicOnError(err, "cannot create JAFAR_INPUT chain") - err = shellx.Run(log.Log, "sudo", "iptables", "-N", "JAFAR_OUTPUT") - runtimex.PanicOnError(err, "cannot create JAFAR_OUTPUT chain") - err = shellx.Run(log.Log, "sudo", "iptables", "-t", "nat", "-N", "JAFAR_NAT_OUTPUT") - runtimex.PanicOnError(err, "cannot create JAFAR_NAT_OUTPUT chain") - err = shellx.Run(log.Log, "sudo", "iptables", "-I", "OUTPUT", "-j", "JAFAR_OUTPUT") - runtimex.PanicOnError(err, "cannot insert jump to JAFAR_OUTPUT") - err = shellx.Run(log.Log, "sudo", "iptables", "-I", "INPUT", "-j", "JAFAR_INPUT") - runtimex.PanicOnError(err, "cannot insert jump to JAFAR_INPUT") - err = shellx.Run(log.Log, "sudo", "iptables", "-t", "nat", "-I", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT") - runtimex.PanicOnError(err, "cannot insert jump to JAFAR_NAT_OUTPUT") - return nil -} - -func (s *linuxShell) dropIfDestinationEquals(ip string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-d", ip, "-j", "DROP") -} - -func (s *linuxShell) rstIfDestinationEqualsAndIsTCP(ip string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "--proto", "tcp", "-d", ip, - "-j", "REJECT", "--reject-with", "tcp-reset", - ) -} - -func (s *linuxShell) dropIfContainsKeywordHex(keyword string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp", - "--hex-string", keyword, "-j", "DROP", - ) -} - -func (s *linuxShell) dropIfContainsKeyword(keyword string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp", - "--string", keyword, "-j", "DROP", - ) -} - -func (s *linuxShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo", - "kmp", "--hex-string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset", - ) -} - -func (s *linuxShell) rstIfContainsKeywordAndIsTCP(keyword string) error { - return shellx.Run(log.Log, - "sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo", - "kmp", "--string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset", - ) -} - -func (s *linuxShell) hijackDNS(address string) error { - // Hijack any DNS query, like the Vodafone station does when using the - // secure network feature. Our transparent proxies will use DoT, in order - // to bypass this restriction and avoid routing loop. - return shellx.Run(log.Log, - "sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "udp", - "--dport", "53", "-j", "DNAT", "--to", address, - ) -} - -func (s *linuxShell) hijackHTTPS(address string) error { - // We need to whitelist root otherwise the traffic sent by Jafar - // itself will match the rule and loop. - return shellx.Run(log.Log, - "sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp", - "--dport", "443", "-m", "owner", "!", "--uid-owner", "0", - "-j", "DNAT", "--to", address, - ) -} - -func (s *linuxShell) hijackHTTP(address string) error { - // We need to whitelist root otherwise the traffic sent by Jafar - // itself will match the rule and loop. - return shellx.Run(log.Log, - "sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp", - "--dport", "80", "-m", "owner", "!", "--uid-owner", "0", - "-j", "DNAT", "--to", address, - ) -} - -func (s *linuxShell) waive() error { - shellx.RunQuiet("sudo", "iptables", "-D", "OUTPUT", "-j", "JAFAR_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-D", "INPUT", "-j", "JAFAR_INPUT") - shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-D", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_INPUT") - shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_INPUT") - shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-F", "JAFAR_NAT_OUTPUT") - shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-X", "JAFAR_NAT_OUTPUT") - return nil -} - -func newShell() *linuxShell { - return &linuxShell{} -} diff --git a/internal/cmd/jafar/iptables/iptables_unsupported.go b/internal/cmd/jafar/iptables/iptables_unsupported.go deleted file mode 100644 index 5c45412006..0000000000 --- a/internal/cmd/jafar/iptables/iptables_unsupported.go +++ /dev/null @@ -1,45 +0,0 @@ -//go:build !linux - -package iptables - -import "errors" - -type otherwiseShell struct{} - -func (*otherwiseShell) createChains() error { - return errors.New("not implemented") -} -func (*otherwiseShell) dropIfDestinationEquals(ip string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) rstIfDestinationEqualsAndIsTCP(ip string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) dropIfContainsKeywordHex(keyword string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) dropIfContainsKeyword(keyword string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) rstIfContainsKeywordAndIsTCP(keyword string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) hijackDNS(address string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) hijackHTTPS(address string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) hijackHTTP(address string) error { - return errors.New("not implemented") -} -func (*otherwiseShell) waive() error { - return errors.New("not implemented") -} - -func newShell() *otherwiseShell { - return &otherwiseShell{} -} diff --git a/internal/cmd/jafar/main.go b/internal/cmd/jafar/main.go deleted file mode 100644 index 3a94bfd3b2..0000000000 --- a/internal/cmd/jafar/main.go +++ /dev/null @@ -1,289 +0,0 @@ -// Jafar is a censorship simulation tool used for testing OONI. -package main - -import ( - "encoding/pem" - "errors" - "flag" - "fmt" - "net" - "net/http" - "os" - "os/signal" - "strings" - "syscall" - - "golang.org/x/sys/execabs" - - "github.com/apex/log" - "github.com/apex/log/handlers/cli" - "github.com/miekg/dns" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/badproxy" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/flagx" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/httpproxy" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/iptables" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/resolver" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/tlsproxy" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "github.com/ooni/probe-cli/v3/internal/shellx" -) - -var ( - badProxyAddress *string - badProxyAddressTLS *string - badProxyTLSOutputCA *string - - dnsProxyAddress *string - dnsProxyBlock flagx.StringArray - dnsProxyHijack flagx.StringArray - dnsProxyIgnore flagx.StringArray - - httpProxyAddress *string - httpProxyBlock flagx.StringArray - - iptablesDropIP flagx.StringArray - iptablesDropKeywordHex flagx.StringArray - iptablesDropKeyword flagx.StringArray - iptablesHijackDNSTo *string - iptablesHijackHTTPSTo *string - iptablesHijackHTTPTo *string - iptablesResetIP flagx.StringArray - iptablesResetKeywordHex flagx.StringArray - iptablesResetKeyword flagx.StringArray - - mainCh chan os.Signal - mainCommand *string - mainUser *string - - tag *string - - tlsProxyAddress *string - tlsProxyBlock flagx.StringArray - tlsProxyOutboundPort *string - - uncensoredResolverDoH *string -) - -func init() { - // badProxy - badProxyAddress = flag.String( - "bad-proxy-address", "127.0.0.1:7117", - "Address where to listen for TCP connections", - ) - badProxyAddressTLS = flag.String( - "bad-proxy-address-tls", "127.0.0.1:4114", - "Address where to listen for TLS connections", - ) - badProxyTLSOutputCA = flag.String( - "bad-proxy-tls-output-ca", "badproxy.pem", - "File where to write the CA used by the bad proxy", - ) - - // dnsProxy - dnsProxyAddress = flag.String( - "dns-proxy-address", "127.0.0.1:53", - "Address where the DNS proxy should listen", - ) - flag.Var( - &dnsProxyBlock, "dns-proxy-block", - "Register keyword triggering NXDOMAIN censorship", - ) - flag.Var( - &dnsProxyHijack, "dns-proxy-hijack", - "Register keyword triggering redirection to 127.0.0.1", - ) - flag.Var( - &dnsProxyIgnore, "dns-proxy-ignore", - "Register keyword causing the proxy to ignore the query", - ) - - // httpProxy - httpProxyAddress = flag.String( - "http-proxy-address", "127.0.0.1:80", - "Address where the HTTP proxy should listen", - ) - flag.Var( - &httpProxyBlock, "http-proxy-block", - "Register keyword triggering HTTP 451 censorship", - ) - - // iptables - flag.Var( - &iptablesDropIP, "iptables-drop-ip", - "Drop traffic to the specified IP address", - ) - flag.Var( - &iptablesDropKeywordHex, "iptables-drop-keyword-hex", - "Drop traffic containing the specified keyword in hex", - ) - flag.Var( - &iptablesDropKeyword, "iptables-drop-keyword", - "Drop traffic containing the specified keyword", - ) - iptablesHijackDNSTo = flag.String( - "iptables-hijack-dns-to", "", - "Hijack all DNS UDP traffic to the specified endpoint", - ) - iptablesHijackHTTPSTo = flag.String( - "iptables-hijack-https-to", "", - "Hijack all HTTPS traffic to the specified endpoint", - ) - iptablesHijackHTTPTo = flag.String( - "iptables-hijack-http-to", "", - "Hijack all HTTP traffic to the specified endpoint", - ) - flag.Var( - &iptablesResetIP, "iptables-reset-ip", - "Reset TCP/IP traffic to the specified IP address", - ) - flag.Var( - &iptablesResetKeywordHex, "iptables-reset-keyword-hex", - "Reset TCP/IP traffic containing the specified keyword in hex", - ) - flag.Var( - &iptablesResetKeyword, "iptables-reset-keyword", - "Reset TCP/IP traffic containing the specified keyword", - ) - - // main - mainCh = make(chan os.Signal, 1) - signal.Notify( - mainCh, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, - ) - mainCommand = flag.String("main-command", "", "Optional command to execute") - mainUser = flag.String("main-user", "nobody", "Run command as user") - - // tag - tag = flag.String("tag", "", "Add tag to a specific run") - - // tlsProxy - tlsProxyAddress = flag.String( - "tls-proxy-address", "127.0.0.1:443", - "Address where the TCP+TLS proxy should listen", - ) - flag.Var( - &tlsProxyBlock, "tls-proxy-block", - "Register keyword triggering TLS censorship", - ) - tlsProxyOutboundPort = flag.String( - "tls-proxy-outbound-port", "443", - "The outbound port where requests should be proxied", - ) - - // uncensored - uncensoredResolverDoH = flag.String( - "uncensored-resolver-doh", "https://1.1.1.1/dns-query", - "URL of an hopefully uncensored DoH resolver", - ) -} - -func badProxyStart() net.Listener { - proxy := badproxy.NewCensoringProxy() - listener, err := proxy.Start(*badProxyAddress) - runtimex.PanicOnError(err, "proxy.Start failed") - return listener -} - -func badProxyStartTLS() net.Listener { - proxy := badproxy.NewCensoringProxy() - listener, cert, err := proxy.StartTLS(*badProxyAddressTLS) - runtimex.PanicOnError(err, "proxy.StartTLS failed") - err = os.WriteFile(*badProxyTLSOutputCA, pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: cert.Raw, - }), 0644) - runtimex.PanicOnError(err, "os.WriteFile failed") - return listener -} - -func dnsProxyStart(uncensored *uncensored.Client) *dns.Server { - proxy := resolver.NewCensoringResolver( - dnsProxyBlock, dnsProxyHijack, dnsProxyIgnore, uncensored, - ) - server, err := proxy.Start(*dnsProxyAddress) - runtimex.PanicOnError(err, "proxy.Start failed") - return server -} - -func httpProxyStart(uncensored *uncensored.Client) *http.Server { - proxy := httpproxy.NewCensoringProxy(httpProxyBlock, uncensored) - server, _, err := proxy.Start(*httpProxyAddress) - runtimex.PanicOnError(err, "proxy.Start failed") - return server -} - -func iptablesStart() *iptables.CensoringPolicy { - policy := iptables.NewCensoringPolicy() - // For robustness waive the policy so we start afresh - policy.Waive() - policy.DropIPs = iptablesDropIP - policy.DropKeywordsHex = iptablesDropKeywordHex - policy.DropKeywords = iptablesDropKeyword - policy.HijackDNSAddress = *iptablesHijackDNSTo - policy.HijackHTTPSAddress = *iptablesHijackHTTPSTo - policy.HijackHTTPAddress = *iptablesHijackHTTPTo - policy.ResetIPs = iptablesResetIP - policy.ResetKeywordsHex = iptablesResetKeywordHex - policy.ResetKeywords = iptablesResetKeyword - err := policy.Apply() - runtimex.PanicOnError(err, "policy.Apply failed") - return policy -} - -func tlsProxyStart(uncensored *uncensored.Client) net.Listener { - proxy := tlsproxy.NewCensoringProxy(tlsProxyBlock, uncensored, *tlsProxyOutboundPort) - listener, err := proxy.Start(*tlsProxyAddress) - runtimex.PanicOnError(err, "proxy.Start failed") - return listener -} - -func newUncensoredClient() *uncensored.Client { - return uncensored.NewClient(*uncensoredResolverDoH) -} - -func mustx(err error, message string, osExit func(int)) { - if err != nil { - var ( - exitcode = 1 - exiterr *execabs.ExitError - ) - if errors.As(err, &exiterr) { - exitcode = exiterr.ExitCode() - } - log.Errorf("%s", message) - osExit(exitcode) - } -} - -func main() { - flag.Parse() - // TODO(bassosimone): we may want a verbose flag - log.SetLevel(log.InfoLevel) - log.SetHandler(cli.Default) - log.Infof("jafar command line: [%s]", strings.Join(os.Args, ", ")) - log.Infof("jafar tag: %s", *tag) - uncensoredClient := newUncensoredClient() - defer uncensoredClient.CloseIdleConnections() - badlistener := badProxyStart() - defer badlistener.Close() - badtlslistener := badProxyStartTLS() - defer badtlslistener.Close() - dnsproxy := dnsProxyStart(uncensoredClient) - defer dnsproxy.Shutdown() - httpproxy := httpProxyStart(uncensoredClient) - defer httpproxy.Close() - tlslistener := tlsProxyStart(uncensoredClient) - defer tlslistener.Close() - policy := iptablesStart() - var err error - if *mainCommand != "" { - err = shellx.RunCommandLine(log.Log, fmt.Sprintf( - "sudo -u '%s' -- %s", *mainUser, *mainCommand, - )) - } else { - <-mainCh - } - policy.Waive() - mustx(err, "subcommand failed", os.Exit) -} diff --git a/internal/cmd/jafar/main_test.go b/internal/cmd/jafar/main_test.go deleted file mode 100644 index 879cf5bf2a..0000000000 --- a/internal/cmd/jafar/main_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package main - -import ( - "errors" - "os" - "runtime" - "testing" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/iptables" - "github.com/ooni/probe-cli/v3/internal/shellx" -) - -func ensureWeStartOverWithIPTables() { - iptables.NewCensoringPolicy().Waive() -} - -func TestNoCommand(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - if runtime.GOOS != "linux" { - t.Skip("skip test on non Linux systems") - } - ensureWeStartOverWithIPTables() - *dnsProxyAddress = "127.0.0.1:0" - *httpProxyAddress = "127.0.0.1:0" - *tlsProxyAddress = "127.0.0.1:0" - go func() { - mainCh <- os.Interrupt - }() - main() -} - -func TestWithCommand(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - if runtime.GOOS != "linux" { - t.Skip("skip test on non Linux systems") - } - ensureWeStartOverWithIPTables() - *dnsProxyAddress = "127.0.0.1:0" - *httpProxyAddress = "127.0.0.1:0" - *tlsProxyAddress = "127.0.0.1:0" - *mainCommand = "whoami" - defer func() { - *mainCommand = "" - }() - main() -} - -func TestMustx(t *testing.T) { - t.Run("with no error", func(t *testing.T) { - var called int - mustx(nil, "", func(int) { - called++ - }) - if called != 0 { - t.Fatal("should not happen") - } - }) - - t.Run("with non-exit-code error", func(t *testing.T) { - var ( - called int - exitcode int - ) - mustx(errors.New("antani"), "", func(ec int) { - called++ - exitcode = ec - }) - if called != 1 { - t.Fatal("not called?!") - } - if exitcode != 1 { - t.Fatal("unexpected exitcode value") - } - }) - - t.Run("with exit-code error", func(t *testing.T) { - var ( - called int - exitcode int - ) - err := shellx.Run(log.Log, "curl", "-sf", "") // cause exitcode == 3 - mustx(err, "", func(ec int) { - called++ - exitcode = ec - }) - if called != 1 { - t.Fatal("not called?!") - } - if exitcode != 3 { - t.Fatal("unexpected exitcode value") - } - }) -} diff --git a/internal/cmd/jafar/resolver/resolver.go b/internal/cmd/jafar/resolver/resolver.go deleted file mode 100644 index 44eba27cb0..0000000000 --- a/internal/cmd/jafar/resolver/resolver.go +++ /dev/null @@ -1,134 +0,0 @@ -// Package resolver contains a censoring DNS resolver. Most queries are -// answered without censorship, but selected queries could either be -// discarded or replied to with a bogon or NXDOMAIN answer. -package resolver - -import ( - "context" - "net" - "strings" - - "github.com/miekg/dns" -) - -// Resolver resolves domain names. -type Resolver interface { - LookupHost(ctx context.Context, hostname string) ([]string, error) -} - -// CensoringResolver is a censoring resolver. -type CensoringResolver struct { - blocked []string - hijacked []string - ignored []string - lookupHost func(ctx context.Context, host string) ([]string, error) -} - -// NewCensoringResolver creates a new CensoringResolver instance using -// the specified list of keywords to censor. blocked is the list of -// keywords that trigger NXDOMAIN if they appear in a query. hijacked -// is similar but redirects to 127.0.0.1, where the transparent HTTP -// and TLS proxies will pick them up. dnsNetwork and dnsAddress are the -// settings to configure the upstream, non censored DNS. -func NewCensoringResolver( - blocked, hijacked, ignored []string, uncensored Resolver, -) *CensoringResolver { - return &CensoringResolver{ - blocked: blocked, - hijacked: hijacked, - ignored: ignored, - lookupHost: uncensored.LookupHost, - } -} - -func (r *CensoringResolver) roundtrip(rw dns.ResponseWriter, req *dns.Msg) { - name := req.Question[0].Name - addrs, err := r.lookupHost(context.Background(), name) - var ips []net.IP - if err == nil { - for _, addr := range addrs { - if ip := net.ParseIP(addr); ip != nil { - ips = append(ips, ip) - } - } - } - r.reply(rw, req, ips) -} - -func (r *CensoringResolver) reply( - rw dns.ResponseWriter, req *dns.Msg, ips []net.IP, -) { - m := new(dns.Msg) - m.Compress = true - m.MsgHdr.RecursionAvailable = true - m.SetReply(req) - for _, ip := range ips { - ipv6 := strings.Contains(ip.String(), ":") - if !ipv6 && req.Question[0].Qtype == dns.TypeA { - m.Answer = append(m.Answer, &dns.A{ - Hdr: dns.RR_Header{ - Name: req.Question[0].Name, - Rrtype: dns.TypeA, - Class: dns.ClassINET, - Ttl: 0, - }, - A: ip, - }) - } - } - if m.Answer == nil { - m.SetRcode(req, dns.RcodeNameError) - } - rw.WriteMsg(m) -} - -func (r *CensoringResolver) failure(rw dns.ResponseWriter, req *dns.Msg) { - m := new(dns.Msg) - m.Compress = true - m.MsgHdr.RecursionAvailable = true - m.SetRcode(req, dns.RcodeServerFailure) - rw.WriteMsg(m) -} - -// ServeDNS serves a DNS request -func (r *CensoringResolver) ServeDNS(rw dns.ResponseWriter, req *dns.Msg) { - if len(req.Question) < 1 { - r.failure(rw, req) - return - } - name := req.Question[0].Name - for _, pattern := range r.blocked { - if strings.Contains(name, pattern) { - r.reply(rw, req, nil) - return - } - } - for _, pattern := range r.hijacked { - if strings.Contains(name, pattern) { - r.reply(rw, req, []net.IP{net.IPv4(127, 0, 0, 1)}) - return - } - } - for _, pattern := range r.ignored { - if strings.Contains(name, pattern) { - return - } - } - r.roundtrip(rw, req) -} - -// Start starts the DNS resolver -func (r *CensoringResolver) Start(address string) (*dns.Server, error) { - packetconn, err := net.ListenPacket("udp", address) - if err != nil { - return nil, err - } - server := &dns.Server{ - Addr: address, - Handler: r, - Net: "udp", - PacketConn: packetconn, - } - go server.ActivateAndServe() - return server, nil -} diff --git a/internal/cmd/jafar/resolver/resolver_test.go b/internal/cmd/jafar/resolver/resolver_test.go deleted file mode 100644 index 103344dda0..0000000000 --- a/internal/cmd/jafar/resolver/resolver_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package resolver - -import ( - "strings" - "testing" - - "github.com/miekg/dns" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" -) - -func TestPass(t *testing.T) { - server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil) - checkrequest(t, server, "example.com", "success", nil) - killserver(t, server) -} - -func TestBlock(t *testing.T) { - server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil) - checkrequest(t, server, "mia-ps.ooni.io", "blocked", nil) - killserver(t, server) -} - -func TestRedirect(t *testing.T) { - server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil) - checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", nil) - killserver(t, server) -} - -func TestIgnore(t *testing.T) { - if testing.Short() { - t.Skip("skip test in short mode") - } - server := newresolver(t, nil, nil, []string{"ooni.nu"}) - iotimeout := "i/o timeout" - checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", &iotimeout) - killserver(t, server) -} - -func TestLookupFailure(t *testing.T) { - server := newresolver(t, nil, nil, nil) - // we should receive same response as when we're blocked - checkrequest(t, server, "example.antani", "blocked", nil) - killserver(t, server) -} - -func TestFailureNoQuestion(t *testing.T) { - resolver := NewCensoringResolver( - nil, nil, nil, uncensored.NewClient("https://1.1.1.1/dns-query"), - ) - resolver.ServeDNS(&fakeResponseWriter{t: t}, new(dns.Msg)) -} - -func TestListenFailure(t *testing.T) { - resolver := NewCensoringResolver( - nil, nil, nil, uncensored.NewClient("https://1.1.1.1/dns-query"), - ) - server, err := resolver.Start("8.8.8.8:53") - if err == nil { - t.Fatal("expected an error here") - } - if server != nil { - t.Fatal("expected nil server here") - } -} - -func newresolver(t *testing.T, blocked, hijacked, ignored []string) *dns.Server { - resolver := NewCensoringResolver( - blocked, hijacked, ignored, - uncensored.NewClient("https://1.1.1.1/dns-query"), - ) - server, err := resolver.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return server -} - -func killserver(t *testing.T, server *dns.Server) { - err := server.Shutdown() - if err != nil { - t.Fatal(err) - } -} - -func checkrequest( - t *testing.T, server *dns.Server, host string, expectStatus string, - expectErrorSuffix *string, -) { - address := server.PacketConn.LocalAddr().String() - query := newquery(host) - reply, err := dns.Exchange(query, address) - if err != nil { - if expectErrorSuffix != nil && - strings.HasSuffix(err.Error(), *expectErrorSuffix) { - return - } - t.Fatal(err) - } - switch expectStatus { - case "success": - checksuccess(t, reply) - case "hijacked": - checkhijacked(t, reply) - case "blocked": - checkblocked(t, reply) - default: - panic("unexpected value") - } -} - -func checksuccess(t *testing.T, reply *dns.Msg) { - if reply.Rcode != dns.RcodeSuccess { - t.Fatal("unexpected rcode", reply.Rcode) - } - if len(reply.Answer) < 1 { - t.Fatal("too few answers") - } - for _, answer := range reply.Answer { - if rr, ok := answer.(*dns.A); ok { - if rr.A.String() == "127.0.0.1" { - t.Fatal("unexpected hijacked response here") - } - } - } -} - -func checkhijacked(t *testing.T, reply *dns.Msg) { - if reply.Rcode != dns.RcodeSuccess { - t.Fatal("unexpected rcode") - } - if len(reply.Answer) < 1 { - t.Fatal("too few answers") - } - for _, answer := range reply.Answer { - if rr, ok := answer.(*dns.A); ok { - if rr.A.String() != "127.0.0.1" { - t.Fatal("unexpected non-hijacked response here") - } - } - } -} - -func checkblocked(t *testing.T, reply *dns.Msg) { - if reply.Rcode != dns.RcodeNameError { - t.Fatal("unexpected rcode") - } - if len(reply.Answer) >= 1 { - t.Fatal("too many answers") - } -} - -func newquery(name string) *dns.Msg { - query := new(dns.Msg) - query.Id = dns.Id() - query.RecursionDesired = true - query.Question = append(query.Question, dns.Question{ - Name: dns.Fqdn(name), - Qclass: dns.ClassINET, - Qtype: dns.TypeA, - }) - return query -} - -type fakeResponseWriter struct { - dns.ResponseWriter - t *testing.T -} - -func (rw *fakeResponseWriter) WriteMsg(m *dns.Msg) error { - if m.Rcode != dns.RcodeServerFailure { - rw.t.Fatal("unexpected rcode") - } - return nil -} diff --git a/internal/cmd/jafar/tlsproxy/tlsproxy.go b/internal/cmd/jafar/tlsproxy/tlsproxy.go deleted file mode 100644 index 401333da8a..0000000000 --- a/internal/cmd/jafar/tlsproxy/tlsproxy.go +++ /dev/null @@ -1,200 +0,0 @@ -// Package tlsproxy contains a censoring TLS proxy. Most traffic is passed -// through using the SNI to choose the hostname to connect to. Specific offending -// SNIs are censored by returning a TLS alert to the client. -package tlsproxy - -import ( - "context" - "crypto/tls" - "errors" - "net" - "strings" - "sync" - - "github.com/apex/log" -) - -// Dialer establishes network connections -type Dialer interface { - DialContext(ctx context.Context, network, address string) (net.Conn, error) -} - -// CensoringProxy is a censoring TLS proxy -type CensoringProxy struct { - keywords []string - dial func(network, address string) (net.Conn, error) - outboundPort string -} - -// NewCensoringProxy creates a new CensoringProxy instance using -// the specified list of keywords to censor. keywords is the list -// of keywords that trigger censorship if any of them appears in -// the SNI record of a ClientHello. dnsNetwork and dnsAddress are -// settings to configure the upstream, non censored DNS. -func NewCensoringProxy( - keywords []string, uncensored Dialer, outboundPort string, -) *CensoringProxy { - return &CensoringProxy{ - keywords: keywords, - dial: func(network, address string) (net.Conn, error) { - return uncensored.DialContext(context.Background(), network, address) - }, - outboundPort: outboundPort, - } -} - -// handshakeReader is a hack to perform the initial part of the -// TLS handshake so to know the SNI and then replay the bytes of -// this initial part of the handshake with the server. -type handshakeReader struct { - net.Conn - incoming []byte -} - -// Read saves the initial bytes of the handshake such that later -// we can replay the handshake with the real TLS server. -func (c *handshakeReader) Read(b []byte) (int, error) { - count, err := c.Conn.Read(b) - if err == nil { - c.incoming = append(c.incoming, b[:count]...) - } - return count, err -} - -// Write prevents writing on the real connection -func (c *handshakeReader) Write(b []byte) (int, error) { - return 0, errors.New("cannot write on this connection") -} - -// forward forwards left traffic to right -func forward(wg *sync.WaitGroup, left, right net.Conn) { - data := make([]byte, 1<<18) - for { - n, err := left.Read(data) - if err != nil { - break - } - if _, err = right.Write(data[:n]); err != nil { - break - } - } - wg.Done() -} - -// reset closes the connection with a RST segment -func reset(conn net.Conn) { - if tc, ok := conn.(*net.TCPConn); ok { - tc.SetLinger(0) - } - conn.Close() -} - -// alertclose sends a TLS alert and then closes the connection -func alertclose(conn net.Conn) { - alertdata := []byte{ - 21, // alert - 3, // version[0] - 3, // version[1] - 0, // length[0] - 2, // length[1] - 2, // fatal - 80, // internal error - } - conn.Write(alertdata) - conn.Close() -} - -// getsni attempts the handshakeReader hack to obtain the SNI by reading -// the beginning of the TLS handshake. On success a nonempty SNI string -// is returned. Otherwise we cannot distinguish between the absence of a -// SNI and any other reading network error that may have occurred. -func getsni(conn *handshakeReader) string { - var ( - sni string - mutex sync.Mutex // just for safety - ) - tls.Server(conn, &tls.Config{ - GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - mutex.Lock() - sni = info.ServerName - mutex.Unlock() - return nil, errors.New("tlsproxy: we can't really continue handshake") - }, - }).Handshake() - return sni -} - -func (p *CensoringProxy) connectingToMyself(conn net.Conn) bool { - local := conn.LocalAddr().String() - localAddr, _, localErr := net.SplitHostPort(local) - remote := conn.RemoteAddr().String() - remoteAddr, _, remoteErr := net.SplitHostPort(remote) - return localErr != nil || remoteErr != nil || localAddr == remoteAddr -} - -// handle implements the TLS SNI proxy -func (p *CensoringProxy) handle(clientconn net.Conn) { - hr := &handshakeReader{Conn: clientconn} - sni := getsni(hr) - if sni == "" { - log.Warn("tlsproxy: network failure or SNI not provided") - reset(clientconn) - return - } - for _, pattern := range p.keywords { - if strings.Contains(sni, pattern) { - log.Warnf("tlsproxy: reject SNI by policy: %s", sni) - alertclose(clientconn) - return - } - } - serverconn, err := p.dial("tcp", net.JoinHostPort(sni, p.outboundPort)) - if err != nil { - log.WithError(err).Warn("tlsproxy: p.dial failed") - alertclose(clientconn) - return - } - if p.connectingToMyself(serverconn) { - log.Warn("tlsproxy: connecting to myself") - alertclose(clientconn) - return - } - if _, err := serverconn.Write(hr.incoming); err != nil { - log.WithError(err).Warn("tlsproxy: serverconn.Write failed") - alertclose(clientconn) - return - } - log.Debugf("tlsproxy: routing for %s", sni) - defer clientconn.Close() - defer serverconn.Close() - var wg sync.WaitGroup - wg.Add(2) - go forward(&wg, clientconn, serverconn) - go forward(&wg, serverconn, clientconn) - wg.Wait() -} - -func (p *CensoringProxy) run(listener net.Listener) { - for { - conn, err := listener.Accept() - if err != nil && strings.Contains( - err.Error(), "use of closed network connection") { - return - } - if err == nil { - // It's difficult to make accept fail, so restructure - // the code such that we enter into the happy path - go p.handle(conn) - } - } -} - -// Start starts the censoring proxy. -func (p *CensoringProxy) Start(address string) (net.Listener, error) { - listener, err := net.Listen("tcp", address) - if err != nil { - return nil, err - } - go p.run(listener) - return listener, nil -} diff --git a/internal/cmd/jafar/tlsproxy/tlsproxy_test.go b/internal/cmd/jafar/tlsproxy/tlsproxy_test.go deleted file mode 100644 index 0887ca4efd..0000000000 --- a/internal/cmd/jafar/tlsproxy/tlsproxy_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package tlsproxy - -import ( - "crypto/tls" - "errors" - "net" - "sync" - "testing" - - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" -) - -func TestPass(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), true, &tls.Config{ - ServerName: "example.com", - }) - killproxy(t, listener) -} - -func TestBlock(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "api.ooni.io", - }) - killproxy(t, listener) -} - -func TestNoSNI(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "", - }) - killproxy(t, listener) -} - -func TestInvalidDomain(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "antani.local", - }) - killproxy(t, listener) -} - -func TestFailHandshake(t *testing.T) { - listener := newproxy(t, "ooni.io") - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "expired.badssl.com", - }) - killproxy(t, listener) -} - -func TestFailConnectingToSelf(t *testing.T) { - proxy := &CensoringProxy{ - dial: func(network string, address string) (net.Conn, error) { - return &mockedConnWriteError{}, nil - }, - } - listener, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - if listener == nil { - t.Fatal("expected non nil listener here") - } - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "www.google.com", - }) - killproxy(t, listener) -} - -func TestFailWriteAfterConnect(t *testing.T) { - proxy := &CensoringProxy{ - dial: func(network string, address string) (net.Conn, error) { - return &mockedConnWriteError{ - // must be different or it refuses connecting to self - localIP: net.IPv4(127, 0, 0, 1), - remoteIP: net.IPv4(127, 0, 0, 2), - }, nil - }, - } - listener, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - if listener == nil { - t.Fatal("expected non nil listener here") - } - checkdialtls(t, listener.Addr().String(), false, &tls.Config{ - ServerName: "www.google.com", - }) - killproxy(t, listener) -} - -func TestListenError(t *testing.T) { - proxy := NewCensoringProxy( - []string{""}, uncensored.NewClient("https://1.1.1.1/dns-query"), "443", - ) - listener, err := proxy.Start("8.8.8.8:80") - if err == nil { - t.Fatal("expected an error here") - } - if listener != nil { - t.Fatal("expected nil listener here") - } -} - -func newproxy(t *testing.T, blocked string) net.Listener { - proxy := NewCensoringProxy( - []string{blocked}, uncensored.NewClient("https://1.1.1.1/dns-query"), "443", - ) - listener, err := proxy.Start("127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - return listener -} - -func killproxy(t *testing.T, listener net.Listener) { - err := listener.Close() - if err != nil { - t.Fatal(err) - } -} - -func checkdialtls( - t *testing.T, proxyAddr string, expectSuccess bool, config *tls.Config, -) { - conn, err := tls.Dial("tcp", proxyAddr, config) - if err != nil && expectSuccess { - t.Fatal(err) - } - if err == nil && !expectSuccess { - t.Fatal("expected failure here") - } - if conn == nil && expectSuccess { - t.Fatal("expected actionable conn") - } - if conn != nil && !expectSuccess { - t.Fatal("expected nil conn") - } - if conn != nil { - conn.Close() - } -} - -type mockedConnWriteError struct { - net.Conn - localIP net.IP - remoteIP net.IP -} - -func (c *mockedConnWriteError) Write(b []byte) (int, error) { - return 0, errors.New("cannot write sorry") -} - -func (c *mockedConnWriteError) LocalAddr() net.Addr { - return &net.TCPAddr{ - IP: c.localIP, - } -} - -func (c *mockedConnWriteError) RemoteAddr() net.Addr { - return &net.TCPAddr{ - IP: c.remoteIP, - } -} - -func TestForwardWriteError(t *testing.T) { - var wg sync.WaitGroup - wg.Add(1) - forward(&wg, &mockedConnReadOkay{}, &mockedConnWriteError{}) -} - -type mockedConnReadOkay struct { - net.Conn -} - -func (c *mockedConnReadOkay) Read(b []byte) (int, error) { - return len(b), nil -} diff --git a/internal/cmd/jafar/uncensored/uncensored.go b/internal/cmd/jafar/uncensored/uncensored.go deleted file mode 100644 index fcb51a7e4a..0000000000 --- a/internal/cmd/jafar/uncensored/uncensored.go +++ /dev/null @@ -1,72 +0,0 @@ -// Package uncensored contains code used by Jafar to evade its own -// censorship efforts by taking alternate routes. -package uncensored - -import ( - "context" - "errors" - "net" - "net/http" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -// Client is DNS, HTTP, and TCP client. -type Client struct { - dnsClient model.Resolver - httpTransport model.HTTPTransport - dialer model.Dialer -} - -// NewClient creates a new Client. -func NewClient(resolverURL string) *Client { - dnsClient := netxlite.NewParallelDNSOverHTTPSResolver(log.Log, resolverURL) - return &Client{ - dnsClient: dnsClient, - httpTransport: netxlite.NewHTTPTransportWithResolver(log.Log, dnsClient), - dialer: netxlite.NewDialerWithResolver(log.Log, dnsClient), - } -} - -// Address implements Resolver.Address -func (c *Client) Address() string { - return c.dnsClient.Address() -} - -// LookupHost implements Resolver.LookupHost -func (c *Client) LookupHost(ctx context.Context, domain string) ([]string, error) { - return c.dnsClient.LookupHost(ctx, domain) -} - -// LookupHTTPS implements model.Resolver.LookupHTTPS. -func (c *Client) LookupHTTPS(ctx context.Context, domain string) (*model.HTTPSSvc, error) { - return nil, errors.New("not implemented") -} - -// LookupNS implements model.Resolver.LookupNS. -func (c *Client) LookupNS(ctx context.Context, domain string) ([]*net.NS, error) { - return nil, errors.New("not implemented") -} - -// Network implements Resolver.Network -func (c *Client) Network() string { - return c.dnsClient.Network() -} - -// DialContext implements Dialer.DialContext -func (c *Client) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - return c.dialer.DialContext(ctx, network, address) -} - -// CloseIdleConnections implement HTTPRoundTripper.CloseIdleConnections -func (c *Client) CloseIdleConnections() { - c.dnsClient.CloseIdleConnections() - c.httpTransport.CloseIdleConnections() -} - -// RoundTrip implement HTTPRoundTripper.RoundTrip -func (c *Client) RoundTrip(req *http.Request) (*http.Response, error) { - return c.httpTransport.RoundTrip(req) -} diff --git a/internal/cmd/jafar/uncensored/uncensored_test.go b/internal/cmd/jafar/uncensored/uncensored_test.go deleted file mode 100644 index d03f229ad3..0000000000 --- a/internal/cmd/jafar/uncensored/uncensored_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package uncensored - -import ( - "bytes" - "context" - "net/http" - "net/url" - "testing" - - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -func TestNewClient(t *testing.T) { - client := NewClient("https://1.1.1.1/dns-query") - defer client.CloseIdleConnections() - if client.Address() != "https://1.1.1.1/dns-query" { - t.Fatal("invalid address") - } - if client.Network() != "doh" { - t.Fatal("invalid network") - } - ctx := context.Background() - addrs, err := client.LookupHost(ctx, "dns.google") - if err != nil { - t.Fatal(err) - } - var quad8, two8two4 bool - for _, addr := range addrs { - quad8 = quad8 || (addr == "8.8.8.8") - two8two4 = two8two4 || (addr == "8.8.4.4") - } - if quad8 != true && two8two4 != true { - t.Fatal("invalid response") - } - conn, err := client.DialContext(ctx, "tcp", "8.8.8.8:853") - if err != nil { - t.Fatal(err) - } - defer conn.Close() - resp, err := client.RoundTrip(&http.Request{ - Method: "GET", - URL: &url.URL{ - Scheme: "https", - Host: "www.google.com", - Path: "/humans.txt", - }, - Header: http.Header{}, - }) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatal("invalid status-code") - } - data, err := netxlite.ReadAllContext(context.Background(), resp.Body) - if err != nil { - t.Fatal(err) - } - if !bytes.HasPrefix(data, []byte("Google is built by a large team")) { - t.Fatal("not the expected body") - } -} diff --git a/internal/cmd/miniooni/README.md b/internal/cmd/miniooni/README.md index c4ab7b6666..0f4d96299d 100644 --- a/internal/cmd/miniooni/README.md +++ b/internal/cmd/miniooni/README.md @@ -2,8 +2,7 @@ This directory contains the source code of a simple CLI client that we use for research as well as for running QA scripts. We designed this tool -to have a CLI similar to MK and OONI Probe v2.x to ease running Jafar -scripts that check whether these tools behave similarly. Perfect backwards +to have a CLI similar to MK and OONI Probe v2.x to ease A/B testing. Perfect backwards compatibility was not a design goal for miniooni. Rather, we aimed to have as little conflict as possible, such that we can run side-by-side QA checks. diff --git a/internal/cmd/tinyjafar/README.md b/internal/cmd/tinyjafar/README.md new file mode 100644 index 0000000000..9b44ad4ceb --- /dev/null +++ b/internal/cmd/tinyjafar/README.md @@ -0,0 +1,113 @@ +# internal/cmd/tinyjafar + +This directory builds a program you can use to provoke simple network +interference conditions, as described in detail below. + +To build, use: + +```console +go build -v ./internal/cmd/tinyjafar +``` + +Any requirement that applies to building OONI Probe also applies to +building this small helper program, since they use the same base library. + +The command line interface is backwards compatible with the one +implemented by [the original jafar](https://github.com/ooni/probe-cli/tree/v3.18.1/internal/cmd/jafar) +except that `tinyjafar` only supports iptables flags. + +To use this tool, you must be on Linux and have iptables installed. We do not +use this tool for QA, but it is mentioned in [tutorials](../../../internal/tutorial/). + +## Drop traffic towards a given IP address + +In one console, run: + +```console +./tinyjafar -iptables-drop-ip 130.192.16.171 +``` + +The program will run some `iptables` commands showing each command +it runs. These commands configure `iptables` to block some internet traffic +and the blocking will stay in place until you interrupt +`tinyjafar` using Ctrl-C. When existing, `tinyjafar` will +undo all the commands it executed when starting up. + +While `tinyjafar` is running, in another console try this command: + +```console +curl -v https://nexa.polito.it/ +``` + +If the IP address has not changed since writing this README, the +`curl` command should eventually timeout when connecting. + +## Drop packets containing an hex sequence + +```console +./tinyjafar -iptables-drop-keyword-hex "|07 65 78 61 6d 70 6c 65 03 63 6f 6d|" +``` + +and + +```console +dig @8.8.8.8 www.example.com +``` + +The `tinyjafar` invocation drops DNS queries for `www.example.com`. + +## Drop packets containing a string + +```console +./tinyjafar -iptables-drop-keyword ooni.org +``` + +and + +```console +curl -v https://ooni.org/ +``` + +We expect cURL to timeout during the TLS handshake since we're +blocking the string that appears in the SNI field. + +## Preventing TCP-connecting to a host + +```console +./tinyjafar -iptables-reset-ip 130.192.16.171 +``` + +and + +```console +curl -v https://nexa.polito.it/ +``` + +This should fail with "connection refused". + +## Resetting a TCP connection containing an hex pattern + +```console +./tinyjafar -iptables-reset-keyword-hex "|6F 6F 6E 69|" +``` + +and + +```console +curl -v https://ooni.org/ +``` + +This should reset the TCP connection because the TLS Client Hello +contains "ooni" (`6F 6F 6E 69` in hex). + +## Resetting a TCP connection containing a string pattern + +`console +./tinyjafar -iptables-reset-keyword ooni +``` + +and + +```console +curl -v https://ooni.org/ +``` diff --git a/internal/cmd/tinyjafar/main.go b/internal/cmd/tinyjafar/main.go new file mode 100644 index 0000000000..b962d2c890 --- /dev/null +++ b/internal/cmd/tinyjafar/main.go @@ -0,0 +1,188 @@ +// Command tinyjafar implements a subset of the CLI flags of the original jafar tool. Because several +// tutorials mention some jafar commands, we want to have a tiny tool to support exploration. +package main + +import ( + "flag" + "fmt" + "io" + "os" + "os/signal" + "sync/atomic" + "syscall" + + "github.com/apex/log" + "github.com/google/shlex" + "github.com/ooni/probe-cli/v3/internal/flagx" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// config contains tinyjafar's configuration. +type config struct { + dropIP flagx.StringArray + dropKeywordHex flagx.StringArray + dropKeyword flagx.StringArray + dryRun bool + resetIP flagx.StringArray + resetKeywordHex flagx.StringArray + resetKeyword flagx.StringArray +} + +func (cfg *config) initFlags(fset *flag.FlagSet) { + fset.Var(&cfg.dropIP, "iptables-drop-ip", "Drop traffic to the specified IP address") + fset.Var(&cfg.dropKeywordHex, "iptables-drop-keyword-hex", "Drop traffic containing the specified keyword in hex") + fset.Var(&cfg.dropKeyword, "iptables-drop-keyword", "Drop traffic containing the specified keyword") + fset.BoolVar(&cfg.dryRun, "dry-run", false, "print which commands we would execute") + fset.Var(&cfg.resetIP, "iptables-reset-ip", "Reset TCP/IP traffic to the specified IP address") + fset.Var(&cfg.resetKeywordHex, "iptables-reset-keyword-hex", "Reset TCP/IP traffic containing the specified keyword in hex") + fset.Var(&cfg.resetKeyword, "iptables-reset-keyword", "Reset TCP/IP traffic containing the specified keyword") +} + +// cmd is a cmd to execute +type cmd struct { + argv []string +} + +// cmdSet contains the commands to execute. The zero value is invalid +// and you must construct using the [newCmdSet] factory. +type cmdSet struct { + setup []*cmd + cleanup []*cmd +} + +func newCmdSet() *cmdSet { + c := &cmdSet{} + + c.addSetupCmd("iptables -N JAFAR_INPUT") + c.addSetupCmd("iptables -N JAFAR_OUTPUT") + c.addSetupCmd("iptables -t nat -N JAFAR_NAT_OUTPUT") + c.addSetupCmd("iptables -I OUTPUT -j JAFAR_OUTPUT") + c.addSetupCmd("iptables -I INPUT -j JAFAR_INPUT") + c.addSetupCmd("iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT") + + addCleanupCmd := func(argv string) { + c.cleanup = append(c.cleanup, &cmd{runtimex.Try1(shlex.Split(argv))}) + } + + addCleanupCmd("iptables -D OUTPUT -j JAFAR_OUTPUT") + addCleanupCmd("iptables -D INPUT -j JAFAR_INPUT") + addCleanupCmd("iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT") + addCleanupCmd("iptables -F JAFAR_INPUT") + addCleanupCmd("iptables -X JAFAR_INPUT") + addCleanupCmd("iptables -F JAFAR_OUTPUT") + addCleanupCmd("iptables -X JAFAR_OUTPUT") + addCleanupCmd("iptables -t nat -F JAFAR_NAT_OUTPUT") + addCleanupCmd("iptables -t nat -X JAFAR_NAT_OUTPUT") + + return c +} + +func (c *cmdSet) addSetupCmd(argv string) { + c.setup = append(c.setup, &cmd{runtimex.Try1(shlex.Split(argv))}) +} + +func (c *cmdSet) handleDropIP(cfg *config) { + for _, ipAddr := range cfg.dropIP { + c.addSetupCmd(fmt.Sprintf("iptables -A JAFAR_OUTPUT -d '%s' -j DROP", ipAddr)) + } +} + +func (c *cmdSet) handleDropKeywordHex(cfg *config) { + for _, keyword := range cfg.dropKeywordHex { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT -m string --algo kmp --hex-string '%s' -j DROP", keyword)) + } +} + +func (c *cmdSet) handleDropKeyword(cfg *config) { + for _, keyword := range cfg.dropKeyword { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT -m string --algo kmp --string '%s' -j DROP", + keyword, + )) + } +} + +func (c *cmdSet) handleResetIP(cfg *config) { + for _, ipAddr := range cfg.resetIP { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT --proto tcp -d '%s' -j REJECT --reject-with tcp-reset", + ipAddr, + )) + } +} + +func (c *cmdSet) handleResetKeywordHex(cfg *config) { + for _, keyword := range cfg.resetKeywordHex { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT -m string --proto tcp --algo kmp --hex-string '%s' -j REJECT --reject-with tcp-reset", + keyword, + )) + } +} + +func (c *cmdSet) handleResetKeyword(cfg *config) { + for _, keyword := range cfg.resetKeyword { + c.addSetupCmd(fmt.Sprintf( + "iptables -A JAFAR_OUTPUT -m string --proto tcp --algo kmp --string '%s' -j REJECT --reject-with tcp-reset", + keyword, + )) + } +} + +func main() { + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGINT) + mainWithArgs(os.Stdout, sigChan, os.Args[1:]...) +} + +var ( + returnImmediately = &atomic.Bool{} + mainWithArgsCalled = &atomic.Int64{} +) + +func mainWithArgs(writer io.Writer, sigChan <-chan os.Signal, args ...string) { + if returnImmediately.Load() { + mainWithArgsCalled.Add(1) + return + } + + cfg := &config{} + fset := flag.NewFlagSet("tinyjafar", flag.ExitOnError) + cfg.initFlags(fset) + + fset.Parse(args) + + cs := newCmdSet() + cs.handleDropIP(cfg) + cs.handleDropKeywordHex(cfg) + cs.handleDropKeyword(cfg) + cs.handleResetIP(cfg) + cs.handleResetKeywordHex(cfg) + cs.handleResetKeyword(cfg) + + // with -dry-run, we're just going to print the commands we'd execute + dryShellRun := func(logger model.Logger, command string, args ...string) error { + _, err := fmt.Fprintf(writer, "+ %s\n", shellx.QuotedCommandLineUnsafe(command, args...)) + return err + } + var runSelector = map[bool]func(logger model.Logger, command string, args ...string) error{ + true: dryShellRun, + false: shellx.Run, + } + runx := runSelector[cfg.dryRun] + + for _, cmd := range cs.setup { + runtimex.Try0(runx(log.Log, cmd.argv[0], cmd.argv[1:]...)) + } + + fmt.Fprintf(writer, "\nUse Ctrl-C to terminate\n\n") + <-sigChan + + for _, cmd := range cs.cleanup { + // ignoring the return value here is intentional to avoid interrupting the cleanup midway + _ = runx(log.Log, cmd.argv[0], cmd.argv[1:]...) + } +} diff --git a/internal/cmd/tinyjafar/main_test.go b/internal/cmd/tinyjafar/main_test.go new file mode 100644 index 0000000000..bcff5dc9f9 --- /dev/null +++ b/internal/cmd/tinyjafar/main_test.go @@ -0,0 +1,254 @@ +package main + +import ( + "os" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestMain(t *testing.T) { + t.Run("test the main function with ReturnImmediately", func(t *testing.T) { + defer returnImmediately.Store(false) + returnImmediately.Store(true) + main() + if mainWithArgsCalled.Load() != 1 { + t.Fatal("main did not call mainWithArguments") + } + }) + + // testcase is a test case for this function + type testcase struct { + // name is the test case name + name string + + // args contains the arguments passed to the command line + args []string + + // expect contains the expected program output + expect []string + } + + testcases := []testcase{ + { + name: "without any command line argument", + args: []string{}, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-drop-ip", + args: []string{ + "-iptables-drop-ip", "130.192.16.171", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -A JAFAR_OUTPUT -d 130.192.16.171 -j DROP", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-drop-keyword-hex", + args: []string{ + "-iptables-drop-keyword-hex", "|07 65 78 61 6d 70 6c 65 03 63 6f 6d|", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + `+ iptables -A JAFAR_OUTPUT -m string --algo kmp --hex-string "|07 65 78 61 6d 70 6c 65 03 63 6f 6d|" -j DROP`, + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-drop-keyword", + args: []string{ + "-iptables-drop-keyword", "ooni.org", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -A JAFAR_OUTPUT -m string --algo kmp --string ooni.org -j DROP", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-reset-ip", + args: []string{ + "-iptables-reset-ip", "130.192.16.171", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -A JAFAR_OUTPUT --proto tcp -d 130.192.16.171 -j REJECT --reject-with tcp-reset", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-reset-keyword-hex", + args: []string{ + "-iptables-reset-keyword-hex", "|6F 6F 6E 69|", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + `+ iptables -A JAFAR_OUTPUT -m string --proto tcp --algo kmp --hex-string "|6F 6F 6E 69|" -j REJECT --reject-with tcp-reset`, + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + + { + name: "with -iptables-reset-keyword", + args: []string{ + "-iptables-reset-keyword", "ooni.org", + }, + expect: []string{ + "+ iptables -N JAFAR_INPUT", + "+ iptables -N JAFAR_OUTPUT", + "+ iptables -t nat -N JAFAR_NAT_OUTPUT", + "+ iptables -I OUTPUT -j JAFAR_OUTPUT", + "+ iptables -I INPUT -j JAFAR_INPUT", + "+ iptables -t nat -I OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -A JAFAR_OUTPUT -m string --proto tcp --algo kmp --string ooni.org -j REJECT --reject-with tcp-reset", + "", + "Use Ctrl-C to terminate", + "", + "+ iptables -D OUTPUT -j JAFAR_OUTPUT", + "+ iptables -D INPUT -j JAFAR_INPUT", + "+ iptables -t nat -D OUTPUT -j JAFAR_NAT_OUTPUT", + "+ iptables -F JAFAR_INPUT", + "+ iptables -X JAFAR_INPUT", + "+ iptables -F JAFAR_OUTPUT", + "+ iptables -X JAFAR_OUTPUT", + "+ iptables -t nat -F JAFAR_NAT_OUTPUT", + "+ iptables -t nat -X JAFAR_NAT_OUTPUT", + "", + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + var builder strings.Builder + input := append([]string{}, tc.args...) + input = append(input, "-dry-run") + + sigChan := make(chan os.Signal) + close(sigChan) // so mainWithArgs would not block + + t.Logf("executing with %+v", input) + mainWithArgs(&builder, sigChan, input...) + + output := builder.String() + lines := strings.Split(output, "\n") + if diff := cmp.Diff(tc.expect, lines); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/internal/engine/.gitignore b/internal/engine/.gitignore index 266f6d781f..1eef38ff89 100644 --- a/internal/engine/.gitignore +++ b/internal/engine/.gitignore @@ -1,16 +1,4 @@ -/*.jsonl -/.vscode -/apitool -/asn.mmdb -/ca-bundle.pem -/country.mmdb /example.org -/jafar -/jafar.exe -/miniooni -/miniooni.exe -/oohelper -/oohelperd /oonipsiphon/ /psiphon-config.json.age /psiphon-config.key diff --git a/internal/experiment/tor/tor_test.go b/internal/experiment/tor/tor_test.go index 9ce8290f81..2aa97b5fa1 100644 --- a/internal/experiment/tor/tor_test.go +++ b/internal/experiment/tor/tor_test.go @@ -815,10 +815,10 @@ func TestTestKeysFillToplevelKeysCoverMissingFields(t *testing.T) { failureString := "eof_error" tk := &TestKeys{ Targets: map[string]TargetResults{ - "foobar": {Failure: &failureString, TargetProtocol: "dir_port"}, - "baz": {TargetProtocol: "dir_port"}, - "jafar": {Failure: &failureString, TargetProtocol: "or_port_dirauth"}, - "jasmine": {TargetProtocol: "or_port_dirauth"}, + "foobar": {Failure: &failureString, TargetProtocol: "dir_port"}, + "baz": {TargetProtocol: "dir_port"}, + "ariel": {Failure: &failureString, TargetProtocol: "or_port_dirauth"}, + "sebastian": {TargetProtocol: "or_port_dirauth"}, }, } tk.fillToplevelKeys() diff --git a/internal/cmd/jafar/flagx/stringarray.go b/internal/flagx/stringarray.go similarity index 100% rename from internal/cmd/jafar/flagx/stringarray.go rename to internal/flagx/stringarray.go diff --git a/internal/cmd/jafar/flagx/stringarray_test.go b/internal/flagx/stringarray_test.go similarity index 88% rename from internal/cmd/jafar/flagx/stringarray_test.go rename to internal/flagx/stringarray_test.go index 256d1ed90d..4b63181375 100644 --- a/internal/cmd/jafar/flagx/stringarray_test.go +++ b/internal/flagx/stringarray_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/ooni/probe-cli/v3/internal/cmd/jafar/flagx" + "github.com/ooni/probe-cli/v3/internal/flagx" ) func TestStringArray(t *testing.T) { @@ -71,3 +71,7 @@ func TestStringArray(t *testing.T) { func assertFlagGetterStringArray(b flagx.StringArray) { func(in flag.Getter) {}(&b) } + +// Make sure the compiler does not complain about an unused function, otherwise next +// time I may end up removing it without noticing it's actually necessary. +var _ = assertFlagGetterStringArray diff --git a/internal/shellx/shellx.go b/internal/shellx/shellx.go index 8d09fbf62b..e47ad4c7fe 100644 --- a/internal/shellx/shellx.go +++ b/internal/shellx/shellx.go @@ -138,7 +138,7 @@ func cmd(config *Config, argv *Argv, envp *Envp) *execabs.Cmd { cmd.Env = append(cmd.Env, entry) } if config.Logger != nil { - cmdline := quotedCommandLineUnsafe(argv.P, argv.V...) + cmdline := QuotedCommandLineUnsafe(argv.P, argv.V...) config.Logger.Infof("+ %s", cmdline) } return cmd @@ -271,9 +271,9 @@ func OutputCommandLine(logger model.Logger, cmdline string) ([]byte, error) { // ErrNoCommandToExecute means that the command line is empty. var ErrNoCommandToExecute = errors.New("shellx: no command to execute") -// quotedCommandLineUnsafe returns a quoted command line. This function is unsafe +// QuotedCommandLineUnsafe returns a quoted command line. This function is unsafe // and SHOULD only be used to produce a nice output. -func quotedCommandLineUnsafe(command string, args ...string) string { +func QuotedCommandLineUnsafe(command string, args ...string) string { v := []string{} v = append(v, maybeQuoteArgUnsafe(command)) for _, a := range args { diff --git a/internal/tutorial/README.md b/internal/tutorial/README.md index 88a80f711b..eaa0e9861c 100644 --- a/internal/tutorial/README.md +++ b/internal/tutorial/README.md @@ -18,7 +18,9 @@ you should check how we do that in [internal/registry](../registry). - [Using the measurex package to write network experiments](measurex): this tutorial explains to you how to use the `measurex` library to write networking code that generates measurements using the OONI data format. You will learn -how to perform DNS, TCP, TLS, QUIC, HTTP, HTTPS, and HTTP3 measurements. +how to perform DNS, TCP, TLS, QUIC, HTTP, HTTPS, and HTTP3 measurements. (Note +that `measurex` is currently deprecated and we need to publish a new tutorial +about an improved approach to measuring, based on a DSL.) - [Low-level networking using netxlite](netxlite): this tutorial introduces you to the `netxlite` networking library. This is the underlying library diff --git a/internal/tutorial/experiment/torsf/chapter01/README.md b/internal/tutorial/experiment/torsf/chapter01/README.md index fe8f4b3821..1c61e55db6 100644 --- a/internal/tutorial/experiment/torsf/chapter01/README.md +++ b/internal/tutorial/experiment/torsf/chapter01/README.md @@ -243,7 +243,7 @@ we will just pretty-print the measurement on the `stdout`. You can now run this code as follows: ``` -$ go run ./experiment/torsf/chapter01 | jq +$ go run ./experiment/torsf/chapter01 | tail -n 1 | jq [snip] { "data_format_version": "", diff --git a/internal/tutorial/experiment/torsf/chapter03/README.md b/internal/tutorial/experiment/torsf/chapter03/README.md index 3ae6719ceb..c1aa8df3fe 100644 --- a/internal/tutorial/experiment/torsf/chapter03/README.md +++ b/internal/tutorial/experiment/torsf/chapter03/README.md @@ -128,7 +128,7 @@ func (m *Measurer) run(ctx context.Context, It's now time to run the new code we've written: ``` -$ go run ./experiment/torsf/chapter03 | jq +$ go run ./experiment/torsf/chapter03 | tail -n 1 | jq 2021/06/21 21:21:18 info [ 0.1%] torsf experiment is running 2021/06/21 21:21:19 info [ 0.2%] torsf experiment is running [...] diff --git a/internal/tutorial/experiment/torsf/chapter04/README.md b/internal/tutorial/experiment/torsf/chapter04/README.md index b90dd53bed..7e58d13992 100644 --- a/internal/tutorial/experiment/torsf/chapter04/README.md +++ b/internal/tutorial/experiment/torsf/chapter04/README.md @@ -163,7 +163,7 @@ so we just close the tunnel and record the bootstrap time. We can now run the code as follows to obtain: ``` -$ go run ./experiment/torsf/chapter04 | jq +$ go run ./experiment/torsf/chapter04 | tail -n 1 | jq [...] Jun 21 23:40:50.000 [notice] Bootstrapped 100% (done): Done 2021/06/21 23:40:50 info [100.0%] torsf experiment is finished diff --git a/internal/tutorial/measurex/README.md b/internal/tutorial/measurex/README.md index f89f9aa476..3f8e71e9b9 100644 --- a/internal/tutorial/measurex/README.md +++ b/internal/tutorial/measurex/README.md @@ -1,5 +1,8 @@ # Using the measurex package to write network experiments +**Note**: measurex is deprecated. We will remove this tutorial +once we have a tutorial for the new DSL-based measurement package. + This tutorial teaches you how to write OONI network experiments using the primitives in the `./internal/measurex` package. The name of this package means either "measure diff --git a/internal/tutorial/measurex/chapter04/README.md b/internal/tutorial/measurex/chapter04/README.md index 20504fb150..5f0664e417 100644 --- a/internal/tutorial/measurex/chapter04/README.md +++ b/internal/tutorial/measurex/chapter04/README.md @@ -221,14 +221,14 @@ To emulate the last two scenarios, if you're on Linux, a possibility is building Jafar with this command: ``` -go build -v ./internal/cmd/jafar +go build -v ./internal/cmd/tinyjafar ``` Then, for example, to provoke a connection reset you can run in a terminal: ``` -sudo ./jafar -iptables-reset-keyword dns.google +sudo ./tinyjafar -iptables-reset-keyword dns.google ``` and you can run this tutorial with `dns.google` as diff --git a/internal/tutorial/measurex/chapter04/main.go b/internal/tutorial/measurex/chapter04/main.go index bf9a2a21d6..25295eafbd 100644 --- a/internal/tutorial/measurex/chapter04/main.go +++ b/internal/tutorial/measurex/chapter04/main.go @@ -222,14 +222,14 @@ func main() { // possibility is building Jafar with this command: // // ``` -// go build -v ./internal/cmd/jafar +// go build -v ./internal/cmd/tinyjafar // ``` // // Then, for example, to provoke a connection reset you // can run in a terminal: // // ``` -// sudo ./jafar -iptables-reset-keyword dns.google +// sudo ./tinyjafar -iptables-reset-keyword dns.google // ``` // // and you can run this tutorial with `dns.google` as diff --git a/internal/tutorial/netxlite/chapter02/README.md b/internal/tutorial/netxlite/chapter02/README.md index 64a5fb5ebf..ef5d9eb2f1 100644 --- a/internal/tutorial/netxlite/chapter02/README.md +++ b/internal/tutorial/netxlite/chapter02/README.md @@ -216,11 +216,11 @@ named `ssl_invalid_hostname`). ### TLS handshake reset -If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) and then run: ```bash -sudo ./jafar -iptables-reset-keyword dns.google +sudo ./tinyjafar -iptables-reset-keyword dns.google ``` Then run in another terminal @@ -233,11 +233,11 @@ Then you can interrupt Jafar using ^C. ### TLS handshake timeout -If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) and then run: ```bash -sudo ./jafar -iptables-drop-keyword dns.google +sudo ./tinyjafar -iptables-drop-keyword dns.google ``` Then run in another terminal diff --git a/internal/tutorial/netxlite/chapter02/main.go b/internal/tutorial/netxlite/chapter02/main.go index 56ca4854e6..76b5dd3f73 100644 --- a/internal/tutorial/netxlite/chapter02/main.go +++ b/internal/tutorial/netxlite/chapter02/main.go @@ -217,11 +217,11 @@ func fatal(err error) { // // ### TLS handshake reset // -// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) // and then run: // // ```bash -// sudo ./jafar -iptables-reset-keyword dns.google +// sudo ./tinyjafar -iptables-reset-keyword dns.google // ``` // // Then run in another terminal @@ -234,11 +234,11 @@ func fatal(err error) { // // ### TLS handshake timeout // -// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) // and then run: // // ```bash -// sudo ./jafar -iptables-drop-keyword dns.google +// sudo ./tinyjafar -iptables-drop-keyword dns.google // ``` // // Then run in another terminal diff --git a/internal/tutorial/netxlite/chapter03/README.md b/internal/tutorial/netxlite/chapter03/README.md index 47f59d4b1d..cd4074be0a 100644 --- a/internal/tutorial/netxlite/chapter03/README.md +++ b/internal/tutorial/netxlite/chapter03/README.md @@ -158,11 +158,11 @@ named `ssl_invalid_hostname`). ### TLS handshake reset -If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) and then run: ```bash -sudo ./jafar -iptables-reset-keyword dns.google +sudo ./tinyjafar -iptables-reset-keyword dns.google ``` Then run in another terminal @@ -175,11 +175,11 @@ Then you can interrupt Jafar using ^C. ### TLS handshake timeout -If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) and then run: ```bash -sudo ./jafar -iptables-drop-keyword dns.google +sudo ./tinyjafar -iptables-drop-keyword dns.google ``` Then run in another terminal diff --git a/internal/tutorial/netxlite/chapter03/main.go b/internal/tutorial/netxlite/chapter03/main.go index 68f5d66d06..a4c38bacac 100644 --- a/internal/tutorial/netxlite/chapter03/main.go +++ b/internal/tutorial/netxlite/chapter03/main.go @@ -159,11 +159,11 @@ func fatal(err error) { // // ### TLS handshake reset // -// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) // and then run: // // ```bash -// sudo ./jafar -iptables-reset-keyword dns.google +// sudo ./tinyjafar -iptables-reset-keyword dns.google // ``` // // Then run in another terminal @@ -176,11 +176,11 @@ func fatal(err error) { // // ### TLS handshake timeout // -// If you're on Linux, build Jafar (`go build -v ./internal/cmd/jafar`) +// If you're on Linux, build Jafar (`go build -v ./internal/cmd/tinyjafar`) // and then run: // // ```bash -// sudo ./jafar -iptables-drop-keyword dns.google +// sudo ./tinyjafar -iptables-drop-keyword dns.google // ``` // // Then run in another terminal diff --git a/script/testjafar.bash b/script/testjafar.bash deleted file mode 100755 index 0c415dcc11..0000000000 --- a/script/testjafar.bash +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - -# -# This script uses cURL to verify that Jafar is able to produce a -# bunch of censorship conditions. It should be noted that this script -# only works on Linux and will never work on other systems. -# - -set -e - -function execute() { - echo "+ $@" 1>&2 - "$@" -} - -function expectexitcode() { - local expect - local exitcode - expect=$1 - shift - set +e - "$@" - exitcode=$? - set -e - echo "expected exitcode $expect, found $exitcode" 1>&2 - if [ $exitcode != $expect ]; then - exit 1 - fi -} - -function runtest() { - echo "=== BEGIN $1 ===" - "$1" - echo "=== END $1 ===" -} - -function http_got_nothing() { - expectexitcode 52 execute ./jafar -iptables-hijack-http-to 127.0.0.1:7117 \ - -main-command 'curl -sm5 --connect-to ::example.com: http://ooni.io' -} - -function http_recv_error() { - expectexitcode 56 execute ./jafar -iptables-reset-keyword ooni \ - -main-command 'curl -sm5 --connect-to ::example.com: http://ooni.io' -} - -function http_operation_timedout() { - expectexitcode 28 execute ./jafar -iptables-drop-keyword ooni \ - -main-command 'curl -sm5 --connect-to ::example.com: http://ooni.io' -} - -function http_couldnt_connect() { - local ip - ip=$(host -tA example.com|cut -f4 -d' ') - expectexitcode 7 execute ./jafar -iptables-reset-ip $ip \ - -main-command 'curl -sm5 --connect-to ::example.com: http://ooni.io' -} - -function http_blockpage() { - outfile=$(mktemp) - chown nobody $outfile # curl runs as user nobody - expectexitcode 0 execute ./jafar -http-proxy-block ooni \ - -iptables-hijack-http-to 127.0.0.1:80 \ - -main-command "curl -so $outfile --connect-to ::example.com: http://ooni.io" - if ! grep -q '451 Unavailable For Legal Reasons' $outfile; then - echo "fatal: the blockpage does not contain the expected pattern" 1>&2 - exit 1 - fi -} - -function dns_injection() { - output=$(expectexitcode 0 execute ./jafar \ - -iptables-hijack-dns-to 127.0.0.1:53 \ - -dns-proxy-hijack ooni \ - -main-command 'dig +time=2 +short @example.com ooni.io') - if [ "$output" != "127.0.0.1" ]; then - echo "fatal: the resulting IP is not the expected one" 1>&2 - exit 1 - fi -} - -function dns_timeout() { - expectexitcode 9 execute ./jafar \ - -iptables-hijack-dns-to 127.0.0.1:53 \ - -dns-proxy-ignore ooni \ - -main-command 'dig +time=2 +short @example.com ooni.io' -} - -function dns_nxdomain() { - output=$(expectexitcode 0 execute ./jafar \ - -iptables-hijack-dns-to 127.0.0.1:53 \ - -dns-proxy-block ooni \ - -main-command 'dig +time=2 +short @example.com ooni.io') - if [ "$output" != "" ]; then - echo "fatal: expected no output here" 1>&2 - exit 1 - fi -} - -function sni_man_in_the_middle() { - expectexitcode 60 execute ./jafar -iptables-hijack-https-to 127.0.0.1:4114 \ - -main-command 'curl -sm5 --connect-to ::example.com: https://ooni.io' -} - -function sni_got_nothing() { - expectexitcode 52 execute ./jafar -iptables-hijack-https-to 127.0.0.1:4114 \ - -main-command 'curl -sm5 --cacert badproxy.pem --connect-to ::example.com: https://ooni.io' -} - -function sni_connect_error() { - expectexitcode 35 execute ./jafar -iptables-reset-keyword ooni \ - -main-command 'curl -sm5 --connect-to ::example.com: https://ooni.io' -} - -function main() { - runtest http_got_nothing - runtest http_recv_error - runtest http_operation_timedout - runtest http_couldnt_connect - runtest http_blockpage - runtest dns_injection - runtest dns_timeout - runtest dns_nxdomain - runtest sni_man_in_the_middle - runtest sni_got_nothing - runtest sni_connect_error -} - -main "$@"