From 853ac947be11a707d8fcb8bb56dbc3076d7ad40f Mon Sep 17 00:00:00 2001 From: Edvinas Gaubas Date: Thu, 10 Aug 2023 12:16:41 +0000 Subject: [PATCH] Integrate LATTE --- .gitlab-ci.yml | 3 + ci/docker/tester/Dockerfile | 9 +- ci/docker/tester/requirements.txt | 1 + ci/jobs/.cond.yml | 4 + ci/jobs/analyse.yml | 4 +- ci/jobs/build.yml | 7 +- ci/jobs/coverage.yml | 62 ++++++++-- ci/jobs/latte.yml | 184 ++++++++++++++++++++++++++++++ ci/jobs/package.yml | 2 +- ci/jobs/test.yml | 121 +++++++++++++------- ci/remote_gitlab_package.py | 111 ++++++++++++++++++ ci/setup_proxy.sh | 14 +++ ci/test_deb.sh | 12 +- magefiles/mage.go | 2 +- test/qa/lib/logging.py | 8 +- test/qa/test_killswitch.py | 21 ++++ test/qa/test_login.py | 24 ---- 17 files changed, 499 insertions(+), 90 deletions(-) create mode 100644 ci/jobs/latte.yml create mode 100644 ci/remote_gitlab_package.py create mode 100755 ci/setup_proxy.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0e6b99d6..742fb304 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ include: - /ci/jobs/package.yml - /ci/jobs/test.yml - /ci/jobs/coverage.yml + - /ci/jobs/latte.yml variables: ARCH: "amd64" @@ -43,6 +44,8 @@ stages: - package - analyse - test + - latte-record + - latte-replay - coverage - installation_tests - upload_packages diff --git a/ci/docker/tester/Dockerfile b/ci/docker/tester/Dockerfile index 8fdfb258..e8e61431 100644 --- a/ci/docker/tester/Dockerfile +++ b/ci/docker/tester/Dockerfile @@ -16,18 +16,21 @@ RUN apt-get update \ # install python packages for tests && python3 -m pip install -r /tmp/requirements.txt \ # install thsark - DEBIAN_FRONTEND=noninteractive apt-get install -y tshark \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y tshark \ # make sure, that Docker does not hang during installation, when we get TUI screen - yes yes | DEBIAN_FRONTEND=teletype dpkg-reconfigure wireshark-common \ + && yes yes | DEBIAN_FRONTEND=teletype dpkg-reconfigure wireshark-common \ # cleanup && apt-get clean ARG USER_ID=1000 ARG GROUP_ID=1000 +# yq is used by LATTE proxy +RUN curl -L -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 \ + && chmod +x /usr/bin/yq + RUN groupadd --system nordvpn && groupadd -g ${GROUP_ID} qa && useradd -l -m -u ${USER_ID} -g qa -G nordvpn qa && echo "qa ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers RUN usermod -a -G wireshark qa USER qa - CMD ["exec", "$@"] diff --git a/ci/docker/tester/requirements.txt b/ci/docker/tester/requirements.txt index dd5aec31..4858dc3a 100644 --- a/ci/docker/tester/requirements.txt +++ b/ci/docker/tester/requirements.txt @@ -22,6 +22,7 @@ pyparsing==2.4.7 pytest==7.2.0 pytest-rerunfailures==10.2.0 pytest-timeout==2.0.1 +python-gitlab == 3.14.0 python-jsonrpc-server==0.4.0 python-language-server==0.36.2 python-lsp-jsonrpc==1.0.0 diff --git a/ci/jobs/.cond.yml b/ci/jobs/.cond.yml index 3beb24a9..bceafe04 100644 --- a/ci/jobs/.cond.yml +++ b/ci/jobs/.cond.yml @@ -16,6 +16,10 @@ rules: # merge request was created with at least one commit or a commit was pushed - if: $CI_PIPELINE_SOURCE == 'merge_request_event' +.cond/proxy-record: + rules: + # manual pipeline was created to record proxy-replay DB + - if: $RECORD == '1' .cond/on-click: rules: # catch all if for basic and merge request pipelines diff --git a/ci/jobs/analyse.yml b/ci/jobs/analyse.yml index 0b45f22c..e6ffb473 100644 --- a/ci/jobs/analyse.yml +++ b/ci/jobs/analyse.yml @@ -99,9 +99,7 @@ analyse/security2: variables: GITLAB_TOKEN: ${CI_JOB_TOKEN} image: !reference [cx-scan, image] - script: - - curl $DISCOVER_IP_URL - - !reference [cx-scan, script] + script: !reference [cx-scan, script] dependencies: [] tags: - mountain-gitlab-runner diff --git a/ci/jobs/build.yml b/ci/jobs/build.yml index 4e0f5e6a..4debec60 100644 --- a/ci/jobs/build.yml +++ b/ci/jobs/build.yml @@ -7,6 +7,7 @@ build/binaries: - !reference [.cond/on-main, rules] - !reference [.cond/on-version-tag, rules] - !reference [.cond/on-merge-request, rules] + - !reference [.cond/proxy-record, rules] variables: BUILD_FLAGS: -cover script: $CI_PROJECT_DIR/ci/compile.sh @@ -40,7 +41,7 @@ build/openvpn: - !reference [.cond/on-main, rules] - !reference [.cond/on-version-tag, rules] - !reference [.cond/on-merge-request, rules] - - !reference [.cond/on-click, rules] + - !reference [.cond/proxy-record, rules] script: $CI_PROJECT_DIR/build/openvpn/build.sh dependencies: [] artifacts: @@ -56,7 +57,7 @@ build/data: - !reference [.cond/on-main, rules] - !reference [.cond/on-version-tag, rules] - !reference [.cond/on-merge-request, rules] - - !reference [.cond/on-click, rules] + - !reference [.cond/proxy-record, rules] script: $CI_PROJECT_DIR/ci/data.sh dependencies: [] artifacts: @@ -69,7 +70,7 @@ build/licenses: - !reference [.cond/on-main, rules] - !reference [.cond/on-version-tag, rules] - !reference [.cond/on-merge-request, rules] - - !reference [.cond/on-click, rules] + - !reference [.cond/proxy-record, rules] script: $CI_PROJECT_DIR/ci/licenses.sh dependencies: [] artifacts: diff --git a/ci/jobs/coverage.yml b/ci/jobs/coverage.yml index 5c759a10..2442b323 100644 --- a/ci/jobs/coverage.yml +++ b/ci/jobs/coverage.yml @@ -1,12 +1,17 @@ include: /ci/jobs/.cond.yml -coverage/integration: - stage: coverage - image: ghcr.io/nordsecurity/nordvpn-linux/builder:1.0.0 +.latte-rules: + rules: + - !reference [.cond/on-merge-request, rules] + +.no-latte-rules: rules: - !reference [.cond/on-main, rules] - !reference [.cond/on-version-tag, rules] - - !reference [.cond/on-merge-request, rules] + +.coverage/integration: + stage: coverage + image: ghcr.io/nordsecurity/nordvpn-linux/builder:1.0.0 script: - $CI_PROJECT_DIR/ci/qa_test_coverage.sh dependencies: @@ -22,13 +27,30 @@ coverage/integration: - test/deb-fileshare allow_failure: true coverage: '/Total coverage: (\d+\.\d+)%/' -coverage/combined: + +coverage/integration-latte: + extends: + - .coverage/integration + - .latte-rules + dependencies: + - latte/deb-replay + - latte/deb-connect1-replay + - latte/deb-connect2-replay + - latte/deb-combinations-replay + - test/deb-manual + - latte/deb-autoconnect1-replay + - latte/deb-autoconnect2-replay + - test/deb-meshnet + - test/deb-fileshare + +coverage/integration-regular: + extends: + - .coverage/integration + - .no-latte-rules + +.coverage/combined: stage: coverage image: ghcr.io/nordsecurity/nordvpn-linux/builder:1.0.0 - rules: - - !reference [.cond/on-main, rules] - - !reference [.cond/on-version-tag, rules] - - !reference [.cond/on-merge-request, rules] script: - $CI_PROJECT_DIR/ci/combined_coverage.sh dependencies: @@ -44,4 +66,24 @@ coverage/combined: - test/deb-meshnet - test/deb-fileshare allow_failure: true - coverage: '/Total coverage: (\d+\.\d+)%/' \ No newline at end of file + coverage: '/Total coverage: (\d+\.\d+)%/' + +coverage/combined-latte: + extends: + - .coverage/combined + - .latte-rules + dependencies: + - latte/deb-replay + - latte/deb-connect1-replay + - latte/deb-connect2-replay + - latte/deb-combinations-replay + - test/deb-manual + - latte/deb-autoconnect1-replay + - latte/deb-autoconnect2-replay + - test/deb-meshnet + - test/deb-fileshare + +coverage/combined-regular: + extends: + - .coverage/combined + - .no-latte-rules \ No newline at end of file diff --git a/ci/jobs/latte.yml b/ci/jobs/latte.yml new file mode 100644 index 00000000..682bd4b9 --- /dev/null +++ b/ci/jobs/latte.yml @@ -0,0 +1,184 @@ +variables: + LATTE_DB_VERSION: v1.0.1 + USER: qa + PROXY_URL: $PROXY_URL + +include: /ci/jobs/.cond.yml + +.job-template: + image: ghcr.io/nordsecurity/nordvpn-linux/tester:1.1.1 + variables: + REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt + LATTE: '1' + rules: + - !reference [.cond/on-merge-request, rules] + dependencies: + - "package/deb: [amd64]" + before_script: + - git clone --branch v1.0.0 https://gitlab-ci-token:${CI_JOB_TOKEN}@$PROXY_URL $CI_PROJECT_DIR/3rd-party/proxy + - cd $CI_PROJECT_DIR/3rd-party/proxy + - ./proxy.sh -i + after_script: + - cp /opt/proxy/databases/_test.db $CI_PROJECT_DIR/dist/recorded.db + - cp /opt/proxy/_dump/proxylogs.log $CI_PROJECT_DIR/proxylogs.log + artifacts: + when: always + paths: + - $CI_PROJECT_DIR/dist/recorded.db + - $CI_PROJECT_DIR/dist/logs/daemon.log + - $CI_PROJECT_DIR/covdatafiles + - $CI_PROJECT_DIR/proxylogs.log + +.record-job-template: + stage: latte-record + extends: .job-template + rules: + - !reference [.cond/proxy-record, rules] + + +.replay-job-template: + stage: latte-replay + extends: .job-template + needs: + - "latte/download-db" + - "package/deb: [amd64]" + dependencies: + - "latte/download-db" + - "package/deb: [amd64]" + + +.setup-proxy-record: + script: + - $CI_PROJECT_DIR/ci/setup_proxy.sh --record + - cd $CI_PROJECT_DIR + +.setup-proxy-replay: + script: + - $CI_PROJECT_DIR/ci/setup_proxy.sh --replay + - cd $CI_PROJECT_DIR + +.copy-record: + script: + - sudo cp $CI_PROJECT_DIR/dist/recorded.db /opt/proxy/databases/_test.db + - sudo chown -R mitmproxyuser:mitmproxyuser /opt/proxy/databases/_test.db + +.copy-replay: + script: + - cp $CI_PROJECT_DIR/dist/latte.db /opt/proxy/databases/_test.db + +.enable_ipv6: + script: + - echo "enable ipv6 (it is needed for transport_test)" + - sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + +latte/download-db: + stage: latte-replay + image: ghcr.io/nordsecurity/nordvpn-linux/tester:1.1.1 + rules: + - !reference [.cond/on-merge-request, rules] + script: + - mkdir $CI_PROJECT_DIR/dist + - | + python3 $CI_PROJECT_DIR/ci/remote_gitlab_package.py \ + download \ + --package-name=LATTE_DB \ + --project=$CI_PROJECT_ID \ + --file=latte.db \ + --version=$LATTE_DB_VERSION \ + --output=$CI_PROJECT_DIR/dist/latte.db + dependencies: [] + artifacts: + when: always + paths: + - $CI_PROJECT_DIR/dist/latte.db + +latte/deb-replay: + extends: .replay-job-template + script: + - !reference [.enable_ipv6, script] + - !reference [.copy-replay, script] + - !reference [.setup-proxy-replay, script] + - $CI_PROJECT_DIR/ci/test_deb.sh $TEST + parallel: + matrix: + - TEST: [connect6, dns, dns6, login, misc, routing, settings] + +latte/deb-autoconnect1-replay: + extends: .replay-job-template + script: + - !reference [.copy-replay, script] + - !reference [.setup-proxy-replay, script] + - $CI_PROJECT_DIR/ci/test_deb.sh autoconnect 'test_autoconnect_default or test_not_autoconnect or test_autoconnect_to_country' + +latte/deb-autoconnect2-replay: + extends: .replay-job-template + script: + - !reference [.copy-replay, script] + - !reference [.setup-proxy-replay, script] + - $CI_PROJECT_DIR/ci/test_deb.sh autoconnect 'test_autoconnect_to_city or test_autoconnect_to_random_server_by_name or test_autoconnect_to_standard_group or test_autoconnect_to_additional_group' + +latte/deb-combinations-replay: + extends: .replay-job-template + script: + - !reference [.copy-replay, script] + - !reference [.setup-proxy-replay, script] + - $CI_PROJECT_DIR/ci/test_deb.sh combinations $PATTERN + parallel: + matrix: + - PATTERN: [test_reconnect_matrix_standard, test_reconnect_matrix_obfuscated, test_connect_country_and_city] + +latte/deb-connect1-replay: + extends: .replay-job-template + script: + - !reference [.copy-replay, script] + - !reference [.setup-proxy-replay, script] + - $CI_PROJECT_DIR/ci/test_deb.sh connect 'test_quick_connect or test_double_quick_connect_only or test_connect_to_absent_server or test_mistype_connect or test_connect_to_invalid_group or test_connect_to_group_flag_standard or test_connect_to_group_flag_additional or test_connect_without_internet_access' + +latte/deb-connect2-replay: + extends: .replay-job-template + script: + - !reference [.copy-replay, script] + - !reference [.setup-proxy-replay, script] + - $CI_PROJECT_DIR/ci/test_deb.sh connect 'test_connect_to_random_server_by_name or test_connection_recovers_from_network_restart or test_double_quick_connect_disconnect or test_connect_to_city or test_connect_to_country or test_connect_to_code_country or test_connect_to_group_standard or test_connect_to_group_additional' + +latte/deb-autoconnect-record: + extends: .record-job-template + script: + - !reference [.setup-proxy-record, script] + - $CI_PROJECT_DIR/ci/test_deb.sh autoconnect + dependencies: + - "package/deb: [amd64]" + needs: + - "package/deb: [amd64]" + +latte/deb-misc-record: + extends: .record-job-template + dependencies: + - "latte/deb-autoconnect-record" + - "package/deb: [amd64]" + needs: + - "latte/deb-autoconnect-record" + - "package/deb: [amd64]" + script: + - !reference [.copy-record, script] + - !reference [.setup-proxy-record, script] + - $CI_PROJECT_DIR/ci/test_deb.sh misc + +latte/upload-db: + stage: latte-record + image: ghcr.io/nordsecurity/nordvpn-linux/tester:1.1.1 + rules: + - !reference [.cond/proxy-record, rules] + dependencies: + - "latte/deb-misc-record" + needs: + - "latte/deb-misc-record" + script: + - mv $CI_PROJECT_DIR/dist/recorded.db $CI_PROJECT_DIR/dist/latte.db + - | + python3 $CI_PROJECT_DIR/ci/remote_gitlab_package.py \ + upload \ + --package-name=LATTE_DB \ + --project=$CI_PROJECT_ID \ + --file=$CI_PROJECT_DIR/dist/latte.db \ + --version=$LATTE_DB_VERSION diff --git a/ci/jobs/package.yml b/ci/jobs/package.yml index bc18caa0..356741ec 100644 --- a/ci/jobs/package.yml +++ b/ci/jobs/package.yml @@ -9,7 +9,7 @@ package/deb: - !reference [.cond/on-main, rules] - !reference [.cond/on-version-tag, rules] - !reference [.cond/on-merge-request, rules] - - !reference [.cond/on-click, rules] + - !reference [.cond/proxy-record, rules] script: $CI_PROJECT_DIR/ci/nfpm/build_packages_resources.sh deb dependencies: - build/data diff --git a/ci/jobs/test.yml b/ci/jobs/test.yml index 7b907b2e..ee4421e8 100644 --- a/ci/jobs/test.yml +++ b/ci/jobs/test.yml @@ -1,15 +1,23 @@ # test.yml defines jobs for testing. include: /ci/jobs/.cond.yml + +.test_ran_with_latte: + rules: + - !reference [.cond/on-main, rules] + - !reference [.cond/on-version-tag, rules] + +.test_not_ran_with_latte: + rules: + - !reference [.cond/on-main, rules] + - !reference [.cond/on-version-tag, rules] + - !reference [.cond/on-merge-request, rules] + .test_job_template: tags: - linux - infra-docker stage: test - image: ghcr.io/nordsecurity/nordvpn-linux/tester:1.1.0 - rules: - - !reference [.cond/on-main, rules] - - !reference [.cond/on-version-tag, rules] - - !reference [.cond/on-merge-request, rules] + image: ghcr.io/nordsecurity/nordvpn-linux/tester:1.1.1 dependencies: - "package/deb: [amd64]" artifacts: @@ -17,18 +25,18 @@ include: /ci/jobs/.cond.yml paths: - $CI_PROJECT_DIR/dist/logs/daemon.log - $CI_PROJECT_DIR/covdatafiles + +.enable_ipv6_test: + script: + - echo "enable ipv6 (it is needed for transport_test)" + - sysctl net.ipv6.conf.all.disable_ipv6=0 + test/cgo: stage: test image: ghcr.io/nordsecurity/nordvpn-linux/builder:1.1.0 - rules: - - !reference [.cond/on-main, rules] - - !reference [.cond/on-version-tag, rules] - - !reference [.cond/on-merge-request, rules] + extends: .test_not_ran_with_latte script: - - echo "is ipv6 enabled?" - - sysctl net.ipv6.conf.all.disable_ipv6 - - echo "enable ipv6 (it is needed for transport_test)" - - sysctl net.ipv6.conf.all.disable_ipv6=0 + - !reference [.enable_ipv6_test, script] - $CI_PROJECT_DIR/ci/test.sh full artifacts: paths: @@ -39,43 +47,66 @@ test/cgo: path: coverage.xml dependencies: [] coverage: '/total:\s*\(statements\)\s*(\d+\.\d+)%/' + test/go: stage: test image: ghcr.io/nordsecurity/nordvpn-linux/builder:1.1.0 - rules: - - !reference [.cond/on-main, rules] - - !reference [.cond/on-version-tag, rules] - - !reference [.cond/on-merge-request, rules] + extends: .test_not_ran_with_latte script: $CI_PROJECT_DIR/ci/test.sh dependencies: [] + test/deb: - extends: .test_job_template - script: - - echo "is ipv6 enabled?" - - sysctl net.ipv6.conf.all.disable_ipv6 - - echo "enable ipv6" - - sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + extends: + - .test_job_template + - .test_ran_with_latte + script: + - !reference [.enable_ipv6_test, script] - $CI_PROJECT_DIR/ci/test_deb.sh $TEST parallel: matrix: - - TEST: [connect6, dns, dns6, killswitch, login, misc, routing, settings, allowlist] + - TEST: [connect6, dns, dns6, login, misc, routing, settings] + +test/deb-non-latte: + extends: + - .test_job_template + - .test_not_ran_with_latte + script: + - $CI_PROJECT_DIR/ci/test_deb.sh $TEST + parallel: + matrix: + - TEST: [killswitch, allowlist] + test/deb-connect1: - extends: .test_job_template - script: $CI_PROJECT_DIR/ci/test_deb.sh connect 'test_quick_connect or test_double_quick_connect_only or test_connect_to_absent_server or test_mistype_connect or test_connect_to_invalid_group or test_connect_to_group_flag_standard or test_connect_to_group_flag_additional or test_connect_without_internet_access or test_connect_to_city or test_connect_to_code_country' + extends: + - .test_job_template + - .test_ran_with_latte + script: $CI_PROJECT_DIR/ci/test_deb.sh connect 'test_quick_connect or test_double_quick_connect_only or test_connect_to_absent_server or test_mistype_connect or test_connect_to_invalid_group or test_connect_to_group_flag_standard or test_connect_to_group_flag_additional or test_connect_without_internet_access' + test/deb-connect2: - extends: .test_job_template - script: $CI_PROJECT_DIR/ci/test_deb.sh connect 'test_connect_to_random_server_by_name or test_connection_recovers_from_network_restart or test_double_quick_connect_disconnect or test_connect_to_country or test_connect_to_group_standard or test_connect_to_group_additional' + extends: + - .test_job_template + - .test_ran_with_latte + script: $CI_PROJECT_DIR/ci/test_deb.sh connect 'test_connect_to_random_server_by_name or test_connection_recovers_from_network_restart or test_double_quick_connect_disconnect or test_connect_to_city or test_connect_to_country or test_connect_to_code_country or test_connect_to_group_standard or test_connect_to_group_additional' + test/deb-combinations: - extends: .test_job_template + extends: + - .test_job_template + - .test_ran_with_latte script: $CI_PROJECT_DIR/ci/test_deb.sh combinations $PATTERN parallel: matrix: - PATTERN: [test_reconnect_matrix_standard, test_reconnect_matrix_obfuscated, test_connect_country_and_city] + test/deb-autoconnect1: - extends: .test_job_template + extends: + - .test_job_template + - .test_ran_with_latte script: $CI_PROJECT_DIR/ci/test_deb.sh autoconnect 'test_autoconnect_default or test_not_autoconnect or test_autoconnect_to_country' + test/deb-autoconnect2: - extends: .test_job_template + extends: + - .test_job_template + - .test_ran_with_latte script: $CI_PROJECT_DIR/ci/test_deb.sh autoconnect 'test_autoconnect_to_city or test_autoconnect_to_random_server_by_name or test_autoconnect_to_standard_group or test_autoconnect_to_additional_group' test/deb-firewall: extends: .test_job_template @@ -88,12 +119,15 @@ test/deb-firewall6: - echo "enable ipv6" - sudo sysctl net.ipv6.conf.all.disable_ipv6=0 - $CI_PROJECT_DIR/ci/test_deb.sh firewall6 + test/deb-manual: - stage: test - image: ghcr.io/nordsecurity/nordvpn-linux/tester:1.1.0 rules: - # TODO: run automatically after meshnet release - !reference [.cond/on-click, rules] + - !reference [.cond/on-main, rules] + - !reference [.cond/on-version-tag, rules] + - !reference [.cond/on-merge-request, rules] + stage: test + image: ghcr.io/nordsecurity/nordvpn-linux/tester:1.1.1 script: $CI_PROJECT_DIR/ci/test_deb.sh $TEST dependencies: - "package/deb: [amd64]" @@ -104,22 +138,24 @@ test/deb-manual: - $CI_PROJECT_DIR/covdatafiles parallel: matrix: - - TEST: [gateway] + - TEST: [gateway, firewall, firewall6] + test/deb-meshnet: - extends: .test_job_template + extends: + - .test_job_template + - .test_not_ran_with_latte variables: QA_PEER_TOKEN: $QA_PEER_TOKEN services: - name: ghcr.io/nordsecurity/nordvpn-linux/qa-peer:1.0.2 alias: qa-peer script: $CI_PROJECT_DIR/ci/test_deb.sh meshnet + test/deb-fileshare: stage: test - image: ghcr.io/nordsecurity/nordvpn-linux/tester:1.1.0 - rules: - - !reference [.cond/on-main, rules] - - !reference [.cond/on-version-tag, rules] - - !reference [.cond/on-merge-request, rules] + image: ghcr.io/nordsecurity/nordvpn-linux/tester:1.1.1 + extends: + - .test_not_ran_with_latte variables: QA_PEER_TOKEN: $QA_PEER_TOKEN services: @@ -140,5 +176,4 @@ test/deb-fileshare: - $CI_PROJECT_DIR/covdatafiles needs: - "package/deb: [amd64]" - - "test/deb-meshnet" - \ No newline at end of file + - "test/deb-meshnet" \ No newline at end of file diff --git a/ci/remote_gitlab_package.py b/ci/remote_gitlab_package.py new file mode 100644 index 00000000..63bcfe6f --- /dev/null +++ b/ci/remote_gitlab_package.py @@ -0,0 +1,111 @@ +from gitlab import Gitlab +from gitlab.v4.objects import Project +import os +import argparse + +CI_JOB_TOKEN = os.environ["CI_JOB_TOKEN"] +GITLAB_URL = os.environ["GITLAB_URL"] + +def upload(project : Project, args): + file_path : str = args.file + file_name = file_path.split("/")[-1] + project.generic_packages.upload( + package_name=args.package_name, + package_version=args.version, + file_name=file_name, + path=file_path + ) + +def download(project : Project, args): + with open (args.output, "wb") as f: + print("downloading..") + project.generic_packages.download( + package_name=args.package_name, + package_version=args.version, + file_name=args.file, + action=f.write, + streamed=True + ) + +def main(args) -> None: + gl = Gitlab(GITLAB_URL, job_token=CI_JOB_TOKEN) + project = gl.projects.get(args.project, lazy=True) + if args.command == "upload": + upload(project, args) + elif args.command == "download": + download(project, args) + else: + print("Unknown command") + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + subparser = parser.add_subparsers(dest="command") + subparser.required = True + upload_parser = subparser.add_parser("upload", help="upload a package") + upload_parser.add_argument( + "--package-name", + type=str, + required=True, + help="package name to use when uploading", + ) + upload_parser.add_argument( + "-f", + "--file", + type=str, + required=True, + help="file path to upload as package", + ) + upload_parser.add_argument( + "-p", + "--project", + type=str, + required=True, + help="ID of GitLab project for uploading packages", + ) + upload_parser.add_argument( + "-v", + "--version", + type=str, + required=True, + help="version to create package for", + ) + download_parser = subparser.add_parser("download", help="download a file from a package") + download_parser.add_argument( + "--package-name", + type=str, + required=True, + help="package name to download from", + ) + download_parser.add_argument( + "-f", + "--file", + type=str, + required=True, + help="file to download from package", + ) + download_parser.add_argument( + "-p", + "--project", + type=str, + required=True, + help="ID of GitLab project for downloading packages", + ) + download_parser.add_argument( + "-v", + "--version", + type=str, + required=True, + help="version to download from", + ) + download_parser.add_argument( + "-o", + "--output", + type=str, + required=True, + help="output file path" + ) + args = parser.parse_args() + + main(args) diff --git a/ci/setup_proxy.sh b/ci/setup_proxy.sh new file mode 100755 index 00000000..1e28d371 --- /dev/null +++ b/ci/setup_proxy.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# --replay or --record +set -eux + +args=$1 + +cd "$CI_PROJECT_DIR"/3rd-party/proxy || exit +./proxy.sh -r latte_config.yml "$args" & +pwd +cd "$CI_PROJECT_DIR" || exit +pwd +"$CI_PROJECT_DIR"/ci/install_deb.sh +sudo mv /var/lib/nordvpn/openvpn /var/lib/nordvpn/openvpn.bak +sudo cp "$CI_PROJECT_DIR"/3rd-party/proxy/pretend_openvpn.sh /var/lib/nordvpn/openvpn \ No newline at end of file diff --git a/ci/test_deb.sh b/ci/test_deb.sh index 82682dba..26b94e57 100755 --- a/ci/test_deb.sh +++ b/ci/test_deb.sh @@ -58,7 +58,17 @@ if ! sudo grep -q "export GOCOVERDIR=${CI_PROJECT_DIR}/${COVERDIR}" "/etc/init.d sudo sed -i "1a export GOCOVERDIR=${CI_PROJECT_DIR}/${COVERDIR}" "/etc/init.d/nordvpn" fi -python3 -m pytest -v --timeout 180 -rsx -x --timeout-method=thread -o log_cli=true "${args[@]}" +if [[ -n ${LATTE:-} ]]; then + if ! sudo grep -q "export IGNORE_HEADER_VALIDATION=1" "/etc/init.d/nordvpn"; then + sudo sed -i "1a export IGNORE_HEADER_VALIDATION=1" "/etc/init.d/nordvpn" + fi + + if ! sudo grep -q "export HTTP_TRANSPORTS=http1" "/etc/init.d/nordvpn"; then + sudo sed -i "1a export HTTP_TRANSPORTS=http1" "/etc/init.d/nordvpn" + fi +fi + +python3 -m pytest -v --timeout 180 -x -rsx --timeout-method=thread -o log_cli=true "${args[@]}" if ! sudo grep -q "export GOCOVERDIR=${CI_PROJECT_DIR}/${COVERDIR}" "/etc/init.d/nordvpn"; then sudo sed -i "2d" "/etc/init.d/nordvpn" diff --git a/magefiles/mage.go b/magefiles/mage.go index 1b4d1867..211a5629 100644 --- a/magefiles/mage.go +++ b/magefiles/mage.go @@ -21,7 +21,7 @@ const ( imagePackager = registryPrefix + "packager:1.0.1" imageProtobufGenerator = registryPrefix + "generator:1.0.1" imageScanner = registryPrefix + "scanner:1.0.0" - imageTester = registryPrefix + "tester:1.1.0" + imageTester = registryPrefix + "tester:1.1.1" imageQAPeer = registryPrefix + "qa-peer:1.0.2" imageLinter = registryPrefix + "linter:1.0.0" imageRuster = registryPrefix + "ruster:1.0.1" diff --git a/test/qa/lib/logging.py b/test/qa/lib/logging.py index 86aec902..f5371278 100644 --- a/test/qa/lib/logging.py +++ b/test/qa/lib/logging.py @@ -8,8 +8,14 @@ def log(data=None): """log test name to the daemon logs or data if provided, but not both""" + # Printing this way prints the pure data into a file, going the bash -c echo route + # is vulnerable to double quotes character being found and subsequent lines being taken + # as pure bash code (and failing as it begins to list processes taking them as commands) if data: - sh.sudo.bash("-c", f"echo \"{data}\" >> {FILE}") + sh.sudo.bash("-c", f"""cat <> {FILE} +{data} +EOF +""") else: test_name = os.environ["PYTEST_CURRENT_TEST"] sh.sudo.bash("-c", f"echo '{test_name}' >> {FILE}") diff --git a/test/qa/test_killswitch.py b/test/qa/test_killswitch.py index 22dd81fb..03bdbf3d 100644 --- a/test/qa/test_killswitch.py +++ b/test/qa/test_killswitch.py @@ -116,3 +116,24 @@ def test_killswitch_reconnect( sh.nordvpn.set.killswitch("off") assert network.is_available() + + +# Test for 3.8.7 hotfix. Account and login commands would not work when killswitch is on +# Issue 441 +def test_fancy_transport(): + sh.nordvpn.logout("--persist-token") + output = sh.nordvpn.set.killswitch("on") + assert "Kill Switch is set to 'enabled' successfully." in output + + output = login.login_as("default") + print(output) + assert "Welcome to NordVPN!" in output + + with lib.ErrorDefer(sh.nordvpn.set.killswitch.off): + output = sh.nordvpn.account() + print(output) + assert "Account Information:" in output + + + sh.nordvpn.set.killswitch("off") + assert network.is_available() \ No newline at end of file diff --git a/test/qa/test_login.py b/test/qa/test_login.py index a0d754a0..53500277 100644 --- a/test/qa/test_login.py +++ b/test/qa/test_login.py @@ -137,27 +137,3 @@ def test_logout_disconnects(): assert "You are logged out." in output assert network.is_disconnected() - -# Test for 3.8.7 hotfix. Account and login commands would not work when killswitch is on -# Issue 441 -def test_fancy_transport(): - output = sh.nordvpn.set.killswitch("on") - assert "Kill Switch is set to 'enabled' successfully." in output - - output = login.login_as("default") - print(output) - assert "Welcome to NordVPN!" in output - - with lib.ErrorDefer(sh.nordvpn.logout): - with lib.ErrorDefer(sh.nordvpn.set.killswitch.off): - output = sh.nordvpn.account() - print(output) - assert "Account Information:" in output - - with lib.ErrorDefer(sh.nordvpn.logout): - sh.nordvpn.set.killswitch("off") - assert network.is_available() - - output = sh.nordvpn.logout("--persist-token") - print(output) - assert "You are logged out." in output