Merge remote-tracking branch 'tor-github/pr/1731'

.travis.yml

@@ -34,55 +34,64 @@ env:
     ## Turn off tor's sandbox in chutney, until we fix sandbox errors that are
     ## triggered by Ubuntu Xenial and Bionic. See #32722.
     - CHUTNEY_TOR_SANDBOX="0"
+    ## The default target for chutney jobs
+    - CHUTNEY_MAKE="test-network-all"
   matrix:
     ## This matrix entry is required, but it doesn't actually create any jobs
     -
 
 matrix:
   ## include creates builds with gcc, linux, unless we override those defaults
   include:
-    ## We run basic tests on macOS
+    ## We run chutney on macOS, because macOS Travis has IPv6
+    ## But we only run the IPv6 chutney tests, to speed up the job
+    - env: CHUTNEY_MAKE="test-network-ipv6" CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
+      os: osx
+
+    ## We also run basic tests on macOS
     - compiler: clang
       os: osx
       ## Turn off some newer features, turn on clang's -Wtypedef-redefinition
       env: C_DIALECT_OPTIONS="-std=gnu99"
-    ## We check NSS
-    ## Use -std=gnu99 to turn off some newer features, and maybe turn on some
-    ## extra gcc warnings?
-    - env: NSS_OPTIONS="--enable-nss" C_DIALECT_OPTIONS="-std=gnu99"
+
     ## We run chutney on Linux, because it's faster than chutney on macOS
     ## Chutney is a fast job, clang is slower on Linux, so we do Chutney clang
     - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
       compiler: clang
-    ## We check asciidoc with distcheck, to make sure we remove doc products
-    ## We use Linux clang, because there are no other Linux clang jobs
-    - env: DISTCHECK="yes" ASCIIDOC_OPTIONS="" SKIP_MAKE_CHECK="yes"
-      compiler: clang
-    ## We include a single coverage build with the best options for coverage
-    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" TOR_TEST_RNG_SEED="636f766572616765"
-    ## We run rust on Linux, because it's faster than rust on macOS
-    ## We check rust offline
-    - env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
+
     ## We check asciidoc with distcheck, to make sure we remove doc products
     - env: DISTCHECK="yes" ASCIIDOC_OPTIONS="" SKIP_MAKE_CHECK="yes"
+
     ## We check disable module relay
     - env: MODULES_OPTIONS="--disable-module-relay"
     ## We check disable module dirauth
     - env: MODULES_OPTIONS="--disable-module-dirauth"
+
+    ## We run rust on Linux, because it's faster than rust on macOS
+    ## We check rust offline
+    - env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
+
+    ## We check NSS
+    ## Use -std=gnu99 to turn off some newer features, and maybe turn on some
+    ## extra gcc warnings?
+    - env: NSS_OPTIONS="--enable-nss" C_DIALECT_OPTIONS="-std=gnu99"
+
+    ## We include a single coverage build with the best options for coverage
+    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" TOR_TEST_RNG_SEED="636f766572616765"
+
+    ## We clone our stem repo and run `make test-stem`
+    - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes"
+
+    ## We run `make doxygen` without `make check`.
+    - env: SKIP_MAKE_CHECK="yes" DOXYGEN="yes"
+
     ## macOS builds are very slow, and we have a limited number of
     ## concurrent macOS jobs. We're not actively developing Rust, so it is
     ## the lowest priority.
     ## We run rust on macOS, because we have seen macOS rust failures before
     #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
     #  compiler: clang
     #  os: osx
-    ## We run chutney on macOS, because macOS Travis has IPv6
-    - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
-      os: osx
-    ## We clone our stem repo and run `make test-stem`
-    - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes"
-    ## We run `make doxygen` without `make check`.
-    - env: SKIP_MAKE_CHECK="yes" DOXYGEN="yes"
 
   ## Allow the build to report success (with non-required sub-builds
   ## continuing to run) if all required sub-builds have succeeded.
@@ -94,11 +103,17 @@ matrix:
     ## macOS rust and chutney are very slow, so we let the build finish before
     ## they are done.  We'd like to fast finish, but still eventually show
     ## any failures in the build status. But Travis doesn't have that ability.
-    - env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
-      compiler: clang
-      os: osx
-    - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
-      os: osx
+
+    ## Since this job is disabled, there's not much point having an exception
+    ## for it
+    #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
+    #  compiler: clang
+    #  os: osx
+
+    ## Since we're actively developing IPv6, we want to require the IPv6
+    ## chutney tests
+    #- env: CHUTNEY_MAKE="test-network-ipv6" CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
+    #  os: osx
 
 ## (Linux only) Use a recent Linux image (Ubuntu Bionic)
 dist: bionic
@@ -230,7 +245,7 @@ script:
   ## We run `make check` because that's what https://jenkins.torproject.org does.
   - if [[ "$SKIP_MAKE_CHECK" == "" ]]; then make check; fi
   - if [[ "$DISTCHECK" != "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_FLAGS"; fi
-  - if [[ "$CHUTNEY" != "" ]]; then make test-network-all; fi
+  - if [[ "$CHUTNEY" != "" ]]; then make "$CHUTNEY_MAKE"; fi
   ## Diagnostic for bug 29437: kill stem if it hangs for 9.5 minutes
   ## Travis will kill the job after 10 minutes with no output
   - if [[ "$TEST_STEM" != "" ]]; then make src/app/tor; timelimit -p -t 540 -s USR1 -T 30 -S ABRT python3 "$STEM_SOURCE_DIR"/run_tests.py --tor src/app/tor --integ --test control.controller --test control.base_controller --test process --log TRACE --log-file stem.log; fi

Makefile.am

@@ -275,60 +275,175 @@ check-local:					\
 	shellcheck				\
 	check-cocci
 
+# test-network requires a copy of Chutney in $CHUTNEY_PATH.
+# Chutney can be cloned from https://git.torproject.org/chutney.git .
+.PHONY: need-chutney-path
 need-chutney-path:
 	@if test ! -d "$$CHUTNEY_PATH"; then \
 		echo '$$CHUTNEY_PATH was not set.'; \
-		if test -d $(top_srcdir)/../chutney -a -x $(top_srcdir)/../chutney/chutney; then \
-			echo "Assuming test-network.sh will find" $(top_srcdir)/../chutney; \
+		if test -d "$(top_srcdir)/../chutney" && \
+		   test -x "$(top_srcdir)/../chutney/chutney"; then \
+			echo "Assuming test-network.sh will find" \
+			  "$(top_srcdir)/../chutney"; \
 		else \
 			echo; \
-			echo "To run these tests, git clone https://git.torproject.org/chutney.git ; export CHUTNEY_PATH=\`pwd\`/chutney"; \
+			echo "To run these tests," \
+			  "git clone" \
+			  "https://git.torproject.org/chutney.git ;" \
+			  "export CHUTNEY_PATH=\`pwd\`/chutney"; \
 			exit 1; \
 		fi \
 	fi
 
-# Note that test-network requires a copy of Chutney in $CHUTNEY_PATH.
-# Chutney can be cloned from https://git.torproject.org/chutney.git .
-test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert
-	$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS)
+# Run some basic tests using automake's test-driver
+.PHONY: test-network
+# Make prints all these recursive invocations by default, but those extra lines
+# break up the chutney flavor statuses
+.SILENT: test-network
+test-network:
+	@$(MAKE) test-network-mkdir
+	@$(MAKE) test-network-clean
+	@$(MAKE) test-network-ipv4-impl \
+	  ipv4_flavors="$(TEST_CHUTNEY_FLAVOR_QUICK)"
+	@$(MAKE) test-network-ipv6-impl \
+	  ipv6_flavors="$(TEST_CHUTNEY_FLAVOR_QUICK_IPV6)"
+	@$(MAKE) test-network-results
 
 # Run all available tests using automake's test-driver
+.PHONY: test-network-all
+.SILENT: test-network-all
+test-network-all:
+	@$(MAKE) test-network-mkdir
+	@$(MAKE) test-network-clean
+	@$(MAKE) test-network-ipv4-impl \
+	  ipv4_flavors="$(TEST_CHUTNEY_FLAVORS)"
+	@$(MAKE) test-network-mixed-impl \
+	  mixed_flavors="$(TEST_CHUTNEY_FLAVORS_MIXED)"
+	@$(MAKE) test-network-ipv6-impl \
+	  ipv6_flavors="$(TEST_CHUTNEY_FLAVORS_IPV6)"
+	@$(MAKE) test-network-results
+
+# Run IPv4 and mixed tests using automake's test-driver
+.PHONY: test-network-ipv4
+.SILENT: test-network-ipv4
+test-network-ipv4:
+	@$(MAKE) test-network-mkdir
+	@$(MAKE) test-network-clean
+	@$(MAKE) test-network-ipv4-impl ipv4_flavors="$(TEST_CHUTNEY_FLAVORS)"
+	@$(MAKE) test-network-mixed-impl \
+	  mixed_flavors="$(TEST_CHUTNEY_FLAVORS_MIXED)"
+	@$(MAKE) test-network-results
+
+# Run IPv6 tests using automake's test-driver
+.PHONY: test-network-ipv6
+.SILENT: test-network-ipv6
+test-network-ipv6:
+	@$(MAKE) test-network-mkdir
+	@$(MAKE) test-network-clean
+	@$(MAKE) test-network-ipv6-impl \
+	  ipv6_flavors="$(TEST_CHUTNEY_FLAVORS_IPV6)"
+	@$(MAKE) test-network-results
+
+# Make the test network log directory, if it does not exist
+.PHONY: test-network-mkdir
+.SILENT: test-network-mkdir
+test-network-mkdir:
+	@mkdir -p "$(TEST_NETWORK_ALL_LOG_DIR)"
+
+# Clean the test network log directory
+.PHONY: test-network-clean
+# We don't silence clean: we want to see the rm command
+# We need to remove all matching files, so we can't quote the glob part of the
+# rm arguments
+test-network-clean:
+	rm -f "$(TEST_NETWORK_ALL_LOG_DIR)"/*.log \
+	      "$(TEST_NETWORK_ALL_LOG_DIR)"/*.trs
+
+# Run the IPv4-only test flavors
+#   - $(ipv4_flavors) contains the tests to run
+.PHONY: test-network-ipv4-impl
+.SILENT: test-network-ipv4-impl
+test-network-ipv4-impl:
+	@$(MAKE) test-network-run flavors="$(ipv4_flavors)" skip_flavors=""
+
+# Run the mixed tests in $(mixed_flavors), if a tor-stable binary is available
+#   - only run mixed tests if we have a tor-stable binary
+#   - $(mixed_flavors) contains the tests to run (or skip)
+.PHONY: test-network-mixed-impl
+.SILENT: test-network-mixed-impl
+test-network-mixed-impl:
+	@flavors=""; \
+	if command -v tor-stable >/dev/null 2>&1; then \
+		echo "tor-stable found, running mixed flavors:" \
+		  "$(mixed_flavors)."; \
+		flavors="$$flavors $(mixed_flavors)"; \
+	else \
+		echo "tor-stable not found, skipping mixed flavors:" \
+		  "$(mixed_flavors)."; \
+		skip_flavors="$$skip_flavors $(mixed_flavors)"; \
+	fi; \
+	$(MAKE) test-network-run flavors="$$flavors" \
+	                         skip_flavors="$$skip_flavors"
+
+# Run the IPv6 tests in $(ipv6_flavors), if IPv6 is available
 #   - only run IPv6 tests if we can ping6 or ping -6 ::1 (localhost)
 #     we try the syntax for BSD ping6, Linux ping6, and Linux ping -6,
 #     because they're incompatible
 #   - some IPv6 tests may fail without an IPv6 DNS server
 #     (see #16971 and #17011)
-#   - only run mixed tests if we have a tor-stable binary
-#   - show tor warnings on the console after each network run
-#     (otherwise, warnings go to the logs, and people don't see them unless
-#     there is a network failure)
-test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert
-	mkdir -p $(TEST_NETWORK_ALL_LOG_DIR)
-	rm -f $(TEST_NETWORK_ALL_LOG_DIR)/*.log $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
-	@flavors="$(TEST_CHUTNEY_FLAVORS)"; \
-	if ping6 -q -c 1 -o ::1 >/dev/null 2>&1 || ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1 || ping -6 -c 1 -W 1 ::1 >/dev/null 2>&1; then \
-		echo "ping6 ::1 or ping ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
-		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
-	else \
-		echo "ping6 ::1 and ping ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
-		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
-	fi; \
-	if command -v tor-stable >/dev/null 2>&1; then \
-		echo "tor-stable found, running mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
-		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
+#   - $(ipv6_flavors) contains the tests to run (or skip)
+.PHONY: test-network-ipv6-impl
+.SILENT: test-network-ipv6-impl
+test-network-ipv6-impl:
+	@flavors=""; \
+	if ping6 -q -c 1 -o   ::1 >/dev/null 2>&1 || \
+	   ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1 || \
+	   ping  -6 -c 1 -W 1 ::1 >/dev/null 2>&1; then \
+		echo "ping6 ::1 or ping ::1 succeeded, running IPv6" \
+		  "flavors: $(ipv6_flavors)."; \
+		flavors="$$flavors $(ipv6_flavors)"; \
 	else \
-		echo "tor-stable not found, skipping mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
-		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
+		echo "ping6 ::1 and ping ::1 failed, skipping IPv6 flavors:" \
+		  "$(ipv6_flavors)."; \
+		skip_flavors="$$skip_flavors $(ipv6_flavors)"; \
 	fi; \
-	for f in $$skip_flavors; do \
+	$(MAKE) test-network-run flavors="$$flavors" \
+	                         skip_flavors="$$skip_flavors"
+
+# Run tests using automake's test-driver
+#   - $(flavors) contains the tests to run
+#   - $(skip_flavors) contains the tests to skip
+.PHONY: test-network-run
+.SILENT: test-network-run
+# We need the word splitting in the "for" lines, so we can't quote
+# $(skip_flavors) or $(flavors)
+test-network-run: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert
+	@for f in $(skip_flavors); do \
 		echo "SKIP: $$f"; \
 	done; \
-	for f in $$flavors; do \
-		$(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
-		$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS); \
-	done; \
-	echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \
-	! grep -q FAIL $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
+	for f in $(flavors); do \
+		$(SHELL) "$(top_srcdir)/test-driver" --test-name "$$f" \
+		  --log-file "$(TEST_NETWORK_ALL_LOG_DIR)/$$f.log" \
+		  --trs-file "$(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs" \
+		  $(TEST_NETWORK_ALL_DRIVER_FLAGS) \
+		  "$(top_srcdir)/src/test/test-network.sh" \
+		    --flavor "$$f" $(TEST_NETWORK_FLAGS); \
+		"$(top_srcdir)/src/test/test-network.sh" \
+		  $(TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS); \
+	done
+
+# Print the results from automake's test-driver
+#   - show tor warnings on the console after each network run
+#     (otherwise, warnings go to the logs, and people don't see them unless
+#     there is a network failure)
+.PHONY: test-network-results
+.SILENT: test-network-results
+# We need to grep all matching files, so we can't quote the glob part of the
+# grep arguments
+test-network-results:
+	@echo \
+	  "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."
+	@! grep -q FAIL "$(TEST_NETWORK_ALL_LOG_DIR)"/*.trs
 
 need-stem-path:
 	@if test ! -d "$$STEM_SOURCE_DIR"; then \

changes/ticket28208

@@ -0,0 +1,4 @@
+  o Testing:
+    - Use the "bridges+hs-v23" chutney network flavor in "make test-network".
+      This test requires a recent version of chutney (mid-February 2020).
+      Closes ticket 28208.

changes/ticket33194

@@ -0,0 +1,4 @@
+  o Testing:
+    - Remove a redundant distcheck job. Closes ticket 33194.
+    - Sort the Travis jobs in order of speed. Putting the slowest jobs first
+      takes full advantage of Travis job concurrency. Closes ticket 33194.

changes/ticket33195

@@ -0,0 +1,4 @@
+  o Testing:
+    - Stop allowing the Chutney IPv6 Travis job to fail. This job was
+      previously configured to fast_finish (which requires allow_failure), to
+      speed up the build. Closes ticket 33195.

changes/ticket33280

@@ -0,0 +1,4 @@
+  o Testing:
+    - Add test-network-ipv4 and test-network-ipv6 jobs to the Makefile.
+      These jobs run the IPv4-only and dual-stack chutney flavours from
+      test-network-all. Closes ticket 33280.

changes/ticket33300

@@ -0,0 +1,3 @@
+  o Testing:
+    - Add a basic IPv6 test to "make test-network". This test only runs when
+      the local machine has an IPv6 stack. Closes ticket 33300.

changes/ticket33303

@@ -0,0 +1,4 @@
+  o Testing:
+    - Run the test-network-ipv6 Makefile target in the Travis CI IPv6 chutney
+      job. This job runs on macOS, so it's a bit slow.
+      Closes ticket 33303.

src/test/include.am

@@ -61,11 +61,21 @@ TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
 	$(TESTSCRIPTS)
 
 # These flavors are run using automake's test-driver and test-network.sh
+
+# run a quick test or two
+# this test only uses IPv4
+TEST_CHUTNEY_FLAVOR_QUICK = bridges+hs-v23
+# only run if we can ping6 ::1 (localhost)
+TEST_CHUTNEY_FLAVOR_QUICK_IPV6 = client-ipv6-only-md
+
+# run a basic set of tests, which only use IPv4
 TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v2-min hs-v3-min \
 	single-onion-v23
+
 # only run if we can ping6 ::1 (localhost)
 TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-v23-ipv6-md \
 	single-onion-v23-ipv6-md
+
 # only run if we can find a stable (or simply another) version of tor
 TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v2