From 4eff308380ae4fe410ae2770620941af7db7d906 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 18 Jul 2023 17:53:03 +0200 Subject: [PATCH] [backport] feat(oonimkall): experimental OONI Run v2 API (#1176) This commit backports 51b694e7ef04fe4770bb1b31aa25f5878ab38988. This commit introduces an experimental OONI Run v2 API, which will simplify the development of Android and iOS OONI Run v2 functionality. We're using the testing host for now (i.e., https://ams-pg-test.ooni.org/). This code is not suitable to be used in production. The API may change without notice in the future. The current API is the result of a trial and error process involving me and @aanorbel. Part of https://github.com/ooni/probe/issues/2503 --- pkg/oonimkall/xoonirun.go | 76 +++++++++++ pkg/oonimkall/xoonirun_internal_test.go | 10 ++ pkg/oonimkall/xoonirun_test.go | 160 ++++++++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 pkg/oonimkall/xoonirun.go create mode 100644 pkg/oonimkall/xoonirun_internal_test.go create mode 100644 pkg/oonimkall/xoonirun_test.go diff --git a/pkg/oonimkall/xoonirun.go b/pkg/oonimkall/xoonirun.go new file mode 100644 index 0000000000..d39898ba7f --- /dev/null +++ b/pkg/oonimkall/xoonirun.go @@ -0,0 +1,76 @@ +package oonimkall + +// +// eXperimental OONI Run code. +// + +import ( + "errors" + "fmt" + "net/http" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// OONIRunFetch fetches a given OONI run descriptor. +// +// The ID argument is the unique identifier of the OONI Run link. For example, in: +// +// https://api.ooni.io/api/_/ooni_run/fetch/297500125102 +// +// The OONI Run link ID is 297500125102. +// +// Warning: this API is currently experimental and we only expose it to facilitate +// developing OONI Run v2. Do not use this API in production. +func (sess *Session) OONIRunFetch(ctx *Context, ID int64) (string, error) { + sess.mtx.Lock() + defer sess.mtx.Unlock() + + // TODO(bassosimone): this code should be changed to use the probeservices.Client + // rather than using an hardcoded URL once we switch to production code. Until then, + // we are going to use the test backend server. + + // For example: https://ams-pg-test.ooni.org/api/_/ooni_run/fetch/297500125102 + URL := &url.URL{ + Scheme: "https", + Opaque: "", + User: nil, + Host: "ams-pg-test.ooni.org", + Path: fmt.Sprintf("/api/_/ooni_run/fetch/%d", ID), + RawPath: "", + OmitHost: false, + ForceQuery: false, + RawQuery: "", + Fragment: "", + RawFragment: "", + } + + return sess.ooniRunFetchWithURLLocked(ctx, URL) +} + +func (sess *Session) ooniRunFetchWithURLLocked(ctx *Context, URL *url.URL) (string, error) { + clnt := sess.sessp.DefaultHTTPClient() + + req, err := http.NewRequestWithContext(ctx.ctx, "GET", URL.String(), nil) + if err != nil { + return "", err + } + + resp, err := clnt.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return "", errors.New("xoonirun: HTTP request failed") + } + + rawResp, err := netxlite.ReadAllContext(ctx.ctx, resp.Body) + if err != nil { + return "", err + } + + return string(rawResp), nil +} diff --git a/pkg/oonimkall/xoonirun_internal_test.go b/pkg/oonimkall/xoonirun_internal_test.go new file mode 100644 index 0000000000..026d25e509 --- /dev/null +++ b/pkg/oonimkall/xoonirun_internal_test.go @@ -0,0 +1,10 @@ +package oonimkall + +import "net/url" + +// OONIRunFetchWithURL is exposed to tests to exercise ooniRunFetchWithURLLocked +func (sess *Session) OONIRunFetchWithURL(ctx *Context, URL *url.URL) (string, error) { + sess.mtx.Lock() + defer sess.mtx.Unlock() + return sess.ooniRunFetchWithURLLocked(ctx, URL) +} diff --git a/pkg/oonimkall/xoonirun_test.go b/pkg/oonimkall/xoonirun_test.go new file mode 100644 index 0000000000..0a1e3d2f04 --- /dev/null +++ b/pkg/oonimkall/xoonirun_test.go @@ -0,0 +1,160 @@ +package oonimkall_test + +import ( + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/netxlite/filtering" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +func TestOONIRunFetch(t *testing.T) { + t.Run("we can fetch a OONI Run link descriptor", func(t *testing.T) { + sess, err := NewSessionForTesting() + if err != nil { + t.Fatal(err) + } + + rawResp, err := sess.OONIRunFetch(sess.NewContext(), 9408643002) + if err != nil { + t.Fatal(err) + } + + expect := map[string]any{ + "creation_time": "2023-07-18T15:38:21Z", + "descriptor": map[string]any{ + "author": "simone@openobservatory.org", + "description": "We use this OONI Run descriptor for writing integration tests for ooni/probe-cli/v3/pkg/oonimkall.", + "description_intl": map[string]any{}, + "icon": "", + "name": "OONIMkAll Integration Testing", + "name_intl": map[string]any{}, + "nettests": []any{ + map[string]any{ + "backend_options": map[string]any{}, + "inputs": []any{string("https://www.example.com/")}, + "is_background_run_enabled": false, + "is_manual_run_enabled": false, + "options": map[string]any{}, + "test_name": "web_connectivity", + }, + }, + "short_description": "Integration testing descriptor for ooni/probe-cli/v3/pkg/oonimkall.", + "short_description_intl": map[string]any{}, + }, + "v": 1.0, + } + + var got map[string]any + runtimex.Try0(json.Unmarshal([]byte(rawResp), &got)) + t.Log(got) + + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we handle the case where the URL is invalid", func(t *testing.T) { + sess, err := NewSessionForTesting() + if err != nil { + t.Fatal(err) + } + + URL := &url.URL{Host: "\t"} // this URL is invalid + + rawResp, err := sess.OONIRunFetchWithURL(sess.NewContext(), URL) + if !strings.HasSuffix(err.Error(), `invalid URL escape "%09"`) { + t.Fatal("unexpected error", err) + } + if rawResp != "" { + t.Fatal("expected empty raw response") + } + }) + + t.Run("we handle the case where the response body is not 200", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + defer server.Close() + + URL := runtimex.Try1(url.Parse(server.URL)) + + sess, err := NewSessionForTesting() + if err != nil { + t.Fatal(err) + } + + rawResp, err := sess.OONIRunFetchWithURL(sess.NewContext(), URL) + if !strings.HasSuffix(err.Error(), "HTTP request failed") { + t.Fatal("unexpected error", err) + } + if rawResp != "" { + t.Fatal("expected empty raw response") + } + }) + + t.Run("we handle the case where the HTTP round trip fails", func(t *testing.T) { + // Implementation note: because we need to backport this patch to the release/3.18 + // branch, it would be quite verbose and burdensome use netem to implement this test, + // since release/3.18 is lagging behind from master in terms of netemx. + server := filtering.NewTLSServer(filtering.TLSActionReset) + defer server.Close() + + URL := &url.URL{ + Scheme: "https", + Host: server.Endpoint(), + Path: "/", + } + + sess, err := NewSessionForTesting() + if err != nil { + t.Fatal(err) + } + + rawResp, err := sess.OONIRunFetchWithURL(sess.NewContext(), URL) + if !strings.HasSuffix(err.Error(), "connection_reset") { + t.Fatal("unexpected error", err) + } + if rawResp != "" { + t.Fatal("expected empty raw response") + } + }) + + t.Run("we handle the case when reading the response body fails", func(t *testing.T) { + // Implementation note: because we need to backport this patch to the release/3.18 + // branch, it would be quite verbose and burdensome use netem to implement this test, + // since release/3.18 is lagging behind from master in terms of netemx. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("{")) + hijacker := w.(http.Hijacker) + conn, _, err := hijacker.Hijack() + runtimex.PanicOnError(err, "hijacker.Hijack failed") + if tc, ok := conn.(*net.TCPConn); ok { + tc.SetLinger(0) + } + conn.Close() + })) + defer server.Close() + + URL := runtimex.Try1(url.Parse(server.URL)) + + sess, err := NewSessionForTesting() + if err != nil { + t.Fatal(err) + } + + rawResp, err := sess.OONIRunFetchWithURL(sess.NewContext(), URL) + if !strings.HasSuffix(err.Error(), "connection_reset") { + t.Fatal("unexpected error", err) + } + if rawResp != "" { + t.Fatal("expected empty raw response") + } + }) +}