diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 61aaf797..e1b56a51 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -30,6 +30,11 @@ functions: working_dir: astrolabe-src command: | ${PYTHON3_BINARY} -m pip install virtualenv + - command: subprocess.exec + params: + working_dir: astrolabe-src + command: | + cat /etc/resolv.conf # Create virtualenv using a CPython 3.5+ binary. - command: subprocess.exec params: @@ -78,11 +83,13 @@ functions: CLUSTER_NAME_SALT: ${build_id} ATLAS_API_USERNAME: ${atlas_key} ATLAS_API_PASSWORD: ${atlas_secret} - ATLAS_API_BASE_URL: ${atlas_url} + ATLAS_API_BASE_URL: https://cloud-dev.mongodb.com/api ATLAS_ORGANIZATION_NAME: ${atlas_organization} + ATLAS_ADMIN_API_USERNAME: ${atlas_admin_api_username} + ATLAS_ADMIN_API_PASSWORD: ${atlas_admin_api_password} add_expansions_to_env: true command: | - astrolabevenv/${PYTHON_BIN_DIR}/astrolabe spec-tests run-one tests/${TEST_NAME}.yaml -e integrations/${DRIVER_DIRNAME}/workload-executor + astrolabevenv/${PYTHON_BIN_DIR}/astrolabe spec-tests run-one tests/${TEST_NAME}.yml -e integrations/${DRIVER_DIRNAME}/workload-executor "validate executor": # Run a MongoDB instance locally. @@ -98,7 +105,7 @@ functions: working_dir: astrolabe-src add_expansions_to_env: true command: | - astrolabevenv/${PYTHON_BIN_DIR}/astrolabe spec-tests validate-workload-executor -e integrations/${DRIVER_DIRNAME}/workload-executor --connection-string "mongodb://localhost:27017/" + astrolabevenv/${PYTHON_BIN_DIR}/astrolabe spec-tests validate-workload-executor -e integrations/${DRIVER_DIRNAME}/workload-executor --connection-string "mongodb://localhost:27017/?serverselectiontimeoutms=10000" "delete test cluster": # Delete the cluster that was used to run the test. @@ -110,11 +117,13 @@ functions: CLUSTER_NAME_SALT: ${build_id} ATLAS_API_USERNAME: ${atlas_key} ATLAS_API_PASSWORD: ${atlas_secret} - ATLAS_API_BASE_URL: ${atlas_url} + ATLAS_API_BASE_URL: https://cloud-dev.mongodb.com/api ATLAS_ORGANIZATION_NAME: ${atlas_organization} + ATLAS_ADMIN_API_USERNAME: ${atlas_admin_api_username} + ATLAS_ADMIN_API_PASSWORD: ${atlas_admin_api_password} add_expansions_to_env: true command: | - astrolabevenv/${PYTHON_BIN_DIR}/astrolabe spec-tests delete-cluster tests/${TEST_NAME}.yaml + astrolabevenv/${PYTHON_BIN_DIR}/astrolabe spec-tests delete-cluster tests/${TEST_NAME}.yml "upload test results": # Upload the xunit-format test results. @@ -122,6 +131,37 @@ functions: params: file: "astrolabe-src/xunit-output/*.xml" + "upload server logs": + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: astrolabe-src/logs.tar.gz + remote_file: ${UPLOAD_BUCKET}/${version_id}/${build_id}-${task_id}-${execution}/server-logs.tar.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: "mongodb-logs.tar.gz" + + "upload event logs": + - command: shell.exec + params: + working_dir: astrolabe-src + continue_on_err: true # Because script may not exist OR platform may not be Windows. + add_expansions_to_env: true + command: | + gzip events.json.gz + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: astrolabe-src/events.json.gz + remote_file: ${UPLOAD_BUCKET}/${version_id}/${build_id}-${task_id}-${execution}/events.json.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: "events.json.gz" + # Functions to run before the test. pre: - func: "install astrolabe" @@ -131,6 +171,8 @@ pre: post: - func: "delete test cluster" - func: "upload test results" + - func: "upload server logs" + - func: "upload event logs" tasks: # Workload executor validation task (patch-only). @@ -140,26 +182,101 @@ tasks: commands: - func: "validate executor" # One test-case per task. + # Use .evergreen/generate-tasks.sh to generate this list. + - name: retryReads-move-sharded + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-move-sharded + - name: retryReads-move + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-move + - name: retryReads-primaryRemoval + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-primaryRemoval + - name: retryReads-primaryTakeover + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-primaryTakeover + - name: retryReads-processRestart-sharded + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-processRestart-sharded + - name: retryReads-processRestart + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-processRestart - name: retryReads-resizeCluster - tags: ["all", "retryReads", "resizeCluster"] + cron: '@weekly' + tags: ["all"] commands: - func: "run test" vars: TEST_NAME: retryReads-resizeCluster + - name: retryReads-testFailover-sharded + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-testFailover-sharded + - name: retryReads-testFailover + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-testFailover - name: retryReads-toggleServerSideJS - tags: ["all", "retryReads", "toggleServerSideJS"] + cron: '@weekly' + tags: ["all"] commands: - func: "run test" vars: TEST_NAME: retryReads-toggleServerSideJS + - name: retryReads-vmRestart-sharded + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-vmRestart-sharded + - name: retryReads-vmRestart + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: retryReads-vmRestart - name: retryWrites-resizeCluster - tags: ["all", "retryWrites", "resizeCluster"] + cron: '@weekly' + tags: ["all"] commands: - func: "run test" vars: TEST_NAME: retryWrites-resizeCluster - name: retryWrites-toggleServerSideJS - tags: ["all", "retryWrites", "toggleServerSideJS"] + cron: '@weekly' + tags: ["all"] commands: - func: "run test" vars: @@ -307,22 +424,6 @@ axes: GOPATH: /home/ubuntu/go buildvariants: -- matrix_name: "tests-python" - matrix_spec: - driver: ["pymongo-master"] - platform: ["ubuntu-18.04"] - runtime: ["python38"] - display_name: "${driver} ${platform} ${runtime}" - tasks: - - ".all" -- matrix_name: "tests-python-windows" - matrix_spec: - driver: ["pymongo-master"] - platform: ["windows-64"] - runtime: ["python37-windows"] - display_name: "${driver} ${platform} ${runtime}" - tasks: - - ".all" - matrix_name: "tests-ruby" matrix_spec: driver: ["ruby-master"] @@ -331,46 +432,65 @@ buildvariants: display_name: "${driver} ${platform} ${runtime}" tasks: - ".all" -- matrix_name: tests-node - matrix_spec: - driver: - - node-master - platform: - - ubuntu-18.04 - runtime: - - node-dubnium - - node-erbium - display_name: "${driver} ${platform} ${runtime}" - tasks: - - .all -- matrix_name: "tests-java" - matrix_spec: - driver: ["java-master"] - platform: ["ubuntu-18.04"] - runtime: ["java11"] - display_name: "${driver} ${platform} ${runtime}" - tasks: - - ".all" -- matrix_name: "tests-dotnet-windows" - matrix_spec: - driver: ["dotnet-master"] - platform: ["windows-64"] - runtime: - - "dotnet-async-netcoreapp2.1" - - "dotnet-sync-netcoreapp2.1" - - "dotnet-async-netcoreapp1.1" - - "dotnet-sync-netcoreapp1.1" - - "dotnet-async-net452" - - "dotnet-sync-net452" - display_name: "${driver} ${platform} ${runtime}" - tasks: - - ".all" -- matrix_name: "tests-go" - matrix_spec: - driver: ["go-master"] - platform: ubuntu-18.04 - runtime: go-13 - display_name: "${driver} ${platform} ${runtime}" - tasks: - - ".all" - +# TODO: re-enable language builds once workload executors have been +# re-implemented to work with the new format +#- matrix_name: "tests-python" +# matrix_spec: +# driver: ["pymongo-master"] +# platform: ["ubuntu-18.04"] +# runtime: ["python27", "python38"] +# display_name: "${driver} ${platform} ${runtime}" +# tasks: +# - ".all" +#- matrix_name: "tests-python-windows" +# matrix_spec: +# driver: ["pymongo-master"] +# platform: ["windows-64"] +# runtime: ["python37-windows"] +# display_name: "${driver} ${platform} ${runtime}" +# tasks: +# - ".all" +#- matrix_name: tests-node +# matrix_spec: +# driver: +# - node-master +# platform: +# - ubuntu-18.04 +# runtime: +# - node-dubnium +# - node-erbium +# display_name: "${driver} ${platform} ${runtime}" +# tasks: +# - .all +#- matrix_name: "tests-java" +# matrix_spec: +# driver: ["java-master"] +# platform: ["ubuntu-18.04"] +# runtime: ["java11"] +# display_name: "${driver} ${platform} ${runtime}" +# tasks: +# - ".all" +#- matrix_name: "tests-dotnet-windows" +# matrix_spec: +# driver: ["dotnet-master"] +# platform: ["windows-64"] +# runtime: +# - "dotnet-async-netcoreapp2.1" +# - "dotnet-sync-netcoreapp2.1" +# - "dotnet-async-netcoreapp1.1" +# - "dotnet-sync-netcoreapp1.1" +# - "dotnet-async-net452" +# - "dotnet-sync-net452" +# display_name: "${driver} ${platform} ${runtime}" +# tasks: +# - ".all" +#- matrix_name: "tests-go" +# matrix_spec: +# driver: ["go-master"] +# platform: ubuntu-18.04 +# runtime: go-13 +# display_name: "${driver} ${platform} ${runtime}" +# tasks: +# - ".all" +# +# diff --git a/.evergreen/generate-tasks.sh b/.evergreen/generate-tasks.sh new file mode 100755 index 00000000..1ee661b8 --- /dev/null +++ b/.evergreen/generate-tasks.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# Use this script to generate the task list for config.yml. + +for f in tests/*.yml; do + task=`basename $f |sed -e s/.yml//` + +cat <<-EOT + - name: $task + cron: '@weekly' + tags: ["all"] + commands: + - func: "run test" + vars: + TEST_NAME: $task +EOT + +done diff --git a/.gitignore b/.gitignore index 5af7bee3..e92e6ece 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ xunit-output/ *.iml integrations/go/go.sum integrations/go/executor +events.json +logs.tar.gz +stats.json +/docs/build diff --git a/astrolabe/cli.py b/astrolabe/cli.py index 830f7681..d4cc43fd 100644 --- a/astrolabe/cli.py +++ b/astrolabe/cli.py @@ -28,6 +28,7 @@ from astrolabe.configuration import ( CONFIGURATION_OPTIONS as CONFIGOPTS, TestCaseConfiguration) from astrolabe.utils import ( + get_logs, create_click_option, get_cluster_name, get_test_name_from_spec_file, ClickLogHandler) from astrolabe.validator import validator_factory @@ -74,17 +75,32 @@ 'the test run. Useful when a test will be run multiple times with ' 'the same cluster name salt.')) +NOCREATE_FLAG = click.option( + '--no-create', is_flag=True, default=False, + help=('Do not create and configure clusters at the beginning of the run ' + 'if they already exist, assume they have already been provisioned by ' + 'a previous run.')) + + +class ContextStore: + def __init__(self, client, admin_client): + self.client = client + self.admin_client = admin_client + @click.group() @create_click_option(CONFIGOPTS.ATLAS_API_BASE_URL) @create_click_option(CONFIGOPTS.ATLAS_API_USERNAME) @create_click_option(CONFIGOPTS.ATLAS_API_PASSWORD) +@create_click_option(CONFIGOPTS.ATLAS_ADMIN_API_USERNAME) +@create_click_option(CONFIGOPTS.ATLAS_ADMIN_API_PASSWORD) @create_click_option(CONFIGOPTS.ATLAS_HTTP_TIMEOUT) @create_click_option(CONFIGOPTS.ASTROLABE_LOGLEVEL) @click.version_option() @click.pass_context def cli(ctx, atlas_base_url, atlas_api_username, - atlas_api_password, http_timeout, log_level): + atlas_api_password, atlas_admin_api_username, atlas_admin_api_password, + http_timeout, log_level): """ Astrolabe is a command-line application for running automated driver @@ -96,7 +112,17 @@ def cli(ctx, atlas_base_url, atlas_api_username, username=atlas_api_username, password=atlas_api_password, timeout=http_timeout) - ctx.obj = client + + if atlas_admin_api_username: + admin_client = AtlasClient( + base_url=atlas_base_url, + username=atlas_admin_api_username, + password=atlas_admin_api_password, + timeout=http_timeout) + else: + admin_client = None + + ctx.obj = ContextStore(client, admin_client) # Configure logging. loglevel = getattr(logging, log_level.upper()) @@ -117,7 +143,9 @@ def cli(ctx, atlas_base_url, atlas_api_username, @click.pass_context def check_connection(ctx): """Command to verify validity of Atlas API credentials.""" - pprint(ctx.obj.root.get().data) + pprint(ctx.obj.client.root.get().data) + if ctx.obj.admin_client: + pprint(ctx.obj.admin_client.root.get().data) @cli.group('organizations') @@ -130,7 +158,7 @@ def atlas_organizations(): @click.pass_context def list_all_organizations(ctx): """List all Atlas Organizations (limited to first 100).""" - pprint(ctx.obj.orgs.get().data) + pprint(ctx.obj.client.orgs.get().data) @atlas_organizations.command('get-one') @@ -140,7 +168,7 @@ def get_one_organization_by_name(ctx, org_name): """Get one Atlas Organization by name. Prints "None" if no organization bearing the given name exists.""" pprint(cmd.get_one_organization_by_name( - client=ctx.obj, organization_name=org_name)) + client=ctx.obj.client, organization_name=org_name)) @cli.group('projects') @@ -156,16 +184,16 @@ def atlas_projects(): def create_project_if_necessary(ctx, org_name, project_name, ): """Ensure that the given Atlas Project exists.""" org = cmd.get_one_organization_by_name( - client=ctx.obj, organization_name=org_name) + client=ctx.obj.client, organization_name=org_name) pprint(cmd.ensure_project( - client=ctx.obj, project_name=project_name, organization_id=org.id)) + client=ctx.obj.client, project_name=project_name, organization_id=org.id)) @atlas_projects.command('list') @click.pass_context def list_projects(ctx): """List all Atlas Projects (limited to first 100).""" - pprint(ctx.obj.groups.get().data) + pprint(ctx.obj.client.groups.get().data) @atlas_projects.command('get-one') @@ -173,7 +201,7 @@ def list_projects(ctx): @click.pass_context def get_one_project_by_name(ctx, project_name): """Get one Atlas Project.""" - pprint(ctx.obj.groups.byName[project_name].get().data) + pprint(ctx.obj.client.groups.byName[project_name].get().data) @atlas_projects.command('enable-anywhere-access') @@ -181,8 +209,8 @@ def get_one_project_by_name(ctx, project_name): @click.pass_context def enable_project_access_from_anywhere(ctx, project_name): """Add 0.0.0.0/0 to the IP whitelist of the Atlas Project.""" - project = ctx.obj.groups.byName[project_name].get().data - cmd.ensure_connect_from_anywhere(client=ctx.obj, project_id=project.id) + project = ctx.obj.client.groups.byName[project_name].get().data + cmd.ensure_connect_from_anywhere(client=ctx.obj.client, project_id=project.id) @cli.group('users') @@ -199,9 +227,9 @@ def atlas_users(): def create_user(ctx, db_username, db_password, project_name): """Create an Atlas User with admin privileges. Modifies user permissions, if the user already exists.""" - project = ctx.obj.groups.byName[project_name].get().data + project = ctx.obj.client.groups.byName[project_name].get().data user = cmd.ensure_admin_user( - client=ctx.obj, project_id=project.id, username=db_username, + client=ctx.obj.client, project_id=project.id, username=db_username, password=db_password) pprint(user) @@ -211,8 +239,8 @@ def create_user(ctx, db_username, db_password, project_name): @click.pass_context def list_users(ctx, project_name): """List all Atlas Users.""" - project = ctx.obj.groups.byName[project_name].get().data - pprint(ctx.obj.groups[project.id].databaseUsers.get().data) + project = ctx.obj.client.groups.byName[project_name].get().data + pprint(ctx.obj.client.groups[project.id].databaseUsers.get().data) @cli.group('clusters') @@ -230,7 +258,7 @@ def atlas_clusters(): @click.pass_context def create_cluster(ctx, project_name, cluster_name, instance_size_name): """Create a new dedicated-tier Atlas Cluster.""" - project = ctx.obj.groups.byName[project_name].get().data + project = ctx.obj.client.groups.byName[project_name].get().data cluster_config = { 'name': cluster_name, @@ -240,7 +268,7 @@ def create_cluster(ctx, project_name, cluster_name, instance_size_name): 'regionName': 'US_WEST_1', 'instanceSizeName': instance_size_name}} - cluster = ctx.obj.groups[project.id].clusters.post(**cluster_config) + cluster = ctx.obj.client.groups[project.id].clusters.post(**cluster_config) pprint(cluster.data) @@ -250,8 +278,8 @@ def create_cluster(ctx, project_name, cluster_name, instance_size_name): @click.pass_context def get_one_cluster_by_name(ctx, cluster_name, project_name): """Get one Atlas Cluster.""" - project = ctx.obj.groups.byName[project_name].get().data - cluster = ctx.obj.groups[project.id].clusters[cluster_name].get() + project = ctx.obj.client.groups.byName[project_name].get().data + cluster = ctx.obj.client.groups[project.id].clusters[cluster_name].get() pprint(cluster.data) @@ -264,7 +292,7 @@ def get_one_cluster_by_name(ctx, cluster_name, project_name): @click.pass_context def resize_cluster(ctx, project_name, cluster_name, instance_size_name): """Resize an existing dedicated-tier Atlas Cluster.""" - project = ctx.obj.groups.byName[project_name].get().data + project = ctx.obj.client.groups.byName[project_name].get().data new_cluster_config = { 'clusterType': 'REPLICASET', @@ -273,7 +301,7 @@ def resize_cluster(ctx, project_name, cluster_name, instance_size_name): 'regionName': 'US_WEST_1', 'instanceSizeName': instance_size_name}} - cluster = ctx.obj.groups[project.id].clusters[cluster_name].patch( + cluster = ctx.obj.client.groups[project.id].clusters[cluster_name].patch( **new_cluster_config) pprint(cluster.data) @@ -284,10 +312,10 @@ def resize_cluster(ctx, project_name, cluster_name, instance_size_name): @click.pass_context def toggle_cluster_javascript(ctx, project_name, cluster_name): """Enable/disable server-side javascript for an existing Atlas Cluster.""" - project = ctx.obj.groups.byName[project_name].get().data + project = ctx.obj.client.groups.byName[project_name].get().data # Alias to reduce verbosity. - pargs = ctx.obj.groups[project.id].clusters[cluster_name].processArgs + pargs = ctx.obj.client.groups[project.id].clusters[cluster_name].processArgs initial_process_args = pargs.get() target_js_value = not initial_process_args.data.javascriptEnabled @@ -301,8 +329,8 @@ def toggle_cluster_javascript(ctx, project_name, cluster_name): @click.pass_context def list_clusters(ctx, project_name): """List all Atlas Clusters.""" - project = ctx.obj.groups.byName[project_name].get().data - clusters = ctx.obj.groups[project.id].clusters.get() + project = ctx.obj.client.groups.byName[project_name].get().data + clusters = ctx.obj.client.groups[project.id].clusters.get() pprint(clusters.data) @@ -312,8 +340,8 @@ def list_clusters(ctx, project_name): @click.pass_context def isready_cluster(ctx, project_name, cluster_name): """Check if the Atlas Cluster is 'IDLE'.""" - project = ctx.obj.groups.byName[project_name].get().data - state = ctx.obj.groups[project.id].clusters[cluster_name].get().data.stateName + project = ctx.obj.client.groups.byName[project_name].get().data + state = ctx.obj.client.groups[project.id].clusters[cluster_name].get().data.stateName if state == "IDLE": click.echo("True") @@ -328,8 +356,8 @@ def isready_cluster(ctx, project_name, cluster_name): @click.pass_context def delete_cluster(ctx, project_name, cluster_name): """Delete the Atlas Cluster.""" - project = ctx.obj.groups.byName[project_name].get().data - ctx.obj.groups[project.id].clusters[cluster_name].delete().data + project = ctx.obj.client.groups.byName[project_name].get().data + ctx.obj.client.groups[project.id].clusters[cluster_name].delete().data click.echo("DONE!") @@ -364,12 +392,13 @@ def spec_tests(): @POLLINGFREQUENCY_OPTION @XUNITOUTPUT_OPTION @NODELETE_FLAG +@NOCREATE_FLAG @EXECUTORSTARTUPTIME_OPTION @click.pass_context def run_single_test(ctx, spec_test_file, workload_executor, db_username, db_password, org_name, project_name, cluster_name_salt, polling_timeout, polling_frequency, - xunit_output, no_delete, startup_time): + xunit_output, no_delete, no_create, startup_time): """ Runs one APM test. This is the main entry point for running APM tests in headless environments. @@ -388,11 +417,13 @@ def run_single_test(ctx, spec_test_file, workload_executor, LOGGER.info(tabulate_astrolabe_configuration(config)) # Step-1: create the Test-Runner. - runner = SingleTestRunner(client=ctx.obj, + runner = SingleTestRunner(client=ctx.obj.client, + admin_client=ctx.obj.admin_client, test_locator_token=spec_test_file, configuration=config, xunit_output=xunit_output, persist_clusters=no_delete, + no_create=no_create, workload_startup_time=startup_time) # Step-2: run the tests. @@ -404,6 +435,37 @@ def run_single_test(ctx, spec_test_file, workload_executor, exit(0) +@spec_tests.command('get-logs') +@click.argument("spec_test_file", type=click.Path( + exists=True, file_okay=True, dir_okay=False, resolve_path=True)) +@ATLASORGANIZATIONNAME_OPTION +@ATLASPROJECTNAME_OPTION +@CLUSTERNAMESALT_OPTION +@POLLINGTIMEOUT_OPTION +@POLLINGFREQUENCY_OPTION +@click.pass_context +def get_logs_cmd(ctx, spec_test_file, org_name, project_name, + cluster_name_salt, polling_timeout, polling_frequency, + ): + """ + Retrieves logs for the cluster and saves them in logs.tar.gz in the + current working directory. + """ + + # Step-1: determine the cluster name for the given test. + cluster_name = get_cluster_name(get_test_name_from_spec_file( + spec_test_file), cluster_name_salt) + + organization = cmd.get_one_organization_by_name( + client=ctx.obj.client, + organization_name=org_name) + project = cmd.ensure_project( + client=ctx.obj.client, project_name=project_name, + organization_id=organization.id) + get_logs(admin_client=ctx.obj.admin_client, + project=project, cluster_name=cluster_name) + + @spec_tests.command('delete-cluster') @click.argument("spec_test_file", type=click.Path( exists=True, file_okay=True, dir_okay=False, resolve_path=True)) @@ -424,11 +486,11 @@ def delete_test_cluster(ctx, spec_test_file, org_name, project_name, # Step-2: delete the cluster. organization = cmd.get_one_organization_by_name( - client=ctx.obj, organization_name=org_name) + client=ctx.obj.client, organization_name=org_name) project = cmd.ensure_project( - client=ctx.obj, project_name=project_name, organization_id=organization.id) + client=ctx.obj.client, project_name=project_name, organization_id=organization.id) try: - ctx.obj.groups[project.id].clusters[cluster_name].delete() + ctx.obj.client.groups[project.id].clusters[cluster_name].delete() except AtlasApiBaseError: pass @@ -470,7 +532,7 @@ def run_headless(ctx, spec_tests_directory, workload_executor, db_username, LOGGER.info(tabulate_astrolabe_configuration(config)) # Step-1: create the Test-Runner. - runner = MultiTestRunner(client=ctx.obj, + runner = MultiTestRunner(client=ctx.obj.client, test_locator_token=spec_tests_directory, configuration=config, xunit_output=xunit_output, @@ -506,5 +568,11 @@ def validate_workload_executor(workload_executor, startup_time, exit(1) +@spec_tests.command() +@click.pass_context +def stats(ctx): + cmd.aggregate_statistics() + + if __name__ == '__main__': cli() diff --git a/astrolabe/commands.py b/astrolabe/commands.py index 8eb63cdc..2dce9e1a 100644 --- a/astrolabe/commands.py +++ b/astrolabe/commands.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import defaultdict import logging +import json from atlasclient import AtlasApiError @@ -88,3 +90,63 @@ def ensure_connect_from_anywhere(*, client, project_id, ): ip_details_list = [{"cidrBlock": "0.0.0.0/0"}] resp = client.groups[project_id].whitelist.post(json=ip_details_list).data LOGGER.debug("Project whitelist details: {}".format(resp)) + + +def aggregate_statistics(): + '''Read the results.json and events.json files, aggregate the events into + statistics and write the statistics into stats.json. + + Statistics calculated: + + - Average command execution time + - 95th percentile command execution time + - 99th percentile command execution time + - Peak number of open connections + ''' + + with open('results.json', 'r') as fp: + stats = json.load(fp) + with open('events.json', 'r') as fp: + events = json.load(fp) + + import numpy + + command_events = [ + event for event in events['events'] + if event['name'].startswith('Command') + ] + map = {} + correlated_events = [] + for event in command_events: + if event['name'] == 'CommandStartedEvent': + map[event['requestId']] = event + else: + started_event = map[event['requestId']] + del map[event['requestId']] + _event = dict(started_event) + _event.update(event) + correlated_events.append(_event) + command_times = [c['duration'] for c in correlated_events] + stats['avgCommandTime'] = numpy.average(command_times) + stats['p95CommandTime'] = numpy.percentile(command_times, 95) + stats['p99CommandTime'] = numpy.percentile(command_times, 99) + + conn_events = [ + event for event in events['events'] + if event['name'].startswith('Connection') or event['name'].startswith('Pool') + ] + counts = defaultdict(lambda: 0) + max_counts = defaultdict(lambda: 0) + conn_count = max_conn_count = 0 + for e in conn_events: + if e['name'] == 'ConnectionCreatedEvent': + counts[e['address']] += 1 + elif e['name'] == 'ConnectionClosedEvent': + counts[e['address']] -= 1 + if counts[e['address']] > max_counts[e['address']]: + max_counts[e['address']] = counts[e['address']] + + stats['maxConnectionCounts'] = max_counts + + with open('stats.json', 'w') as fp: + json.dump(stats, fp) diff --git a/astrolabe/configuration.py b/astrolabe/configuration.py index 35bd4b60..cdcc1f2b 100644 --- a/astrolabe/configuration.py +++ b/astrolabe/configuration.py @@ -36,7 +36,7 @@ 'help': 'Maximum time (in s) to poll API endpoints.', 'cliopt': '--polling-timeout', 'envvar': 'ATLAS_POLLING_TIMEOUT', - 'default': 1200.0}, + 'default': 3600.0}, 'ATLAS_POLLING_FREQUENCY': { 'type': click.FLOAT, 'help': 'Frequency (in Hz) at which to poll API endpoints.', @@ -53,6 +53,16 @@ 'cliopt': '--atlas-api-password', 'envvar': 'ATLAS_API_PASSWORD', 'default': None}, + 'ATLAS_ADMIN_API_USERNAME': { + 'help': 'HTTP-Digest username (Atlas API public-key).', + 'cliopt': '--atlas-admin-api-username', + 'envvar': 'ATLAS_ADMIN_API_USERNAME', + 'default': None}, + 'ATLAS_ADMIN_API_PASSWORD': { + 'help': 'HTTP-Digest password (Atlas API private-key).', + 'cliopt': '--atlas-admin-api-password', + 'envvar': 'ATLAS_ADMIN_API_PASSWORD', + 'default': None}, 'ATLAS_DB_USERNAME': { 'help': 'Database username on the MongoDB instance.', 'cliopt': '--db-username', diff --git a/astrolabe/poller.py b/astrolabe/poller.py index b88668ab..293076bb 100644 --- a/astrolabe/poller.py +++ b/astrolabe/poller.py @@ -45,7 +45,7 @@ def poll(self, objects, *, attribute, args, kwargs): timer.start() while timer.elapsed < self.timeout: logmsg = "Polling {} [elapsed: {:.2f} seconds]" - LOGGER.debug(logmsg.format(objects, timer.elapsed)) + LOGGER.info(logmsg.format(objects, timer.elapsed)) for obj in objects: return_value = self._check_ready(obj, attribute, args, kwargs) if return_value: @@ -53,7 +53,7 @@ def poll(self, objects, *, attribute, args, kwargs): LOGGER.debug("Waiting {:.2f} seconds before retrying".format( self.interval)) sleep(self.interval) - raise PollingTimeoutError + raise PollingTimeoutError("Polling timed out after %s seconds" % self.timeout) class BooleanCallablePoller(PollerBase): diff --git a/astrolabe/runner.py b/astrolabe/runner.py index c0117316..4406db8d 100644 --- a/astrolabe/runner.py +++ b/astrolabe/runner.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging -import os +import logging, datetime, time as _time, gzip +import os, io, re from time import sleep from urllib.parse import urlencode +from pymongo import MongoClient from tabulate import tabulate import junitparser import yaml +from .utils import mongo_client from atlasclient import AtlasApiError, JSONObject from astrolabe.commands import ( get_one_organization_by_name, ensure_project, ensure_admin_user, @@ -29,18 +31,19 @@ from astrolabe.poller import BooleanCallablePoller from astrolabe.utils import ( assert_subset, get_cluster_name, get_test_name_from_spec_file, - load_test_data, DriverWorkloadSubprocessRunner, SingleTestXUnitLogger, - Timer) + DriverWorkloadSubprocessRunner, SingleTestXUnitLogger, + get_logs, Timer) LOGGER = logging.getLogger(__name__) class AtlasTestCase: - def __init__(self, *, client, test_name, cluster_name, specification, + def __init__(self, *, client, admin_client, test_name, cluster_name, specification, configuration): # Initialize. self.client = client + self.admin_client = admin_client self.id = test_name self.cluster_name = cluster_name self.spec = specification @@ -69,19 +72,10 @@ def cluster_url(self): def get_connection_string(self): if self.__connection_string is None: cluster = self.cluster_url.get().data - prefix, suffix = cluster.srvAddress.split("//") - uri_options = self.spec.maintenancePlan.uriOptions.copy() - - # Boolean options must be converted to lowercase strings. - for key, value in uri_options.items(): - if isinstance(value, bool): - uri_options[key] = str(value).lower() - - connection_string = (prefix + "//" + self.config.database_username - + ":" + self.config.database_password + "@" - + suffix + "/?") - connection_string += urlencode(uri_options) - self.__connection_string = connection_string + uri = re.sub(r'://', + '://%s:%s@' % (self.config.database_username, self.config.database_password), + cluster.srvAddress) + self.__connection_string = uri return self.__connection_string def __repr__(self): @@ -91,29 +85,40 @@ def is_cluster_state(self, goal_state): cluster_info = self.cluster_url.get().data return cluster_info.stateName.lower() == goal_state.lower() - def verify_cluster_configuration_matches(self, state): + def verify_cluster_configuration_matches(self, expected_configuration): """Verify that the cluster config is what we expect it to be (based on maintenance status). Raises AssertionError.""" - state = state.lower() - if state not in ("initial", "final"): - raise AstrolabeTestCaseError( - "State must be either 'initial' or 'final'.") cluster_config = self.cluster_url.get().data assert_subset( cluster_config, - self.spec.maintenancePlan[state].clusterConfiguration) + expected_configuration.clusterConfiguration) process_args = self.cluster_url.processArgs.get().data assert_subset( - process_args, self.spec.maintenancePlan[state].processArgs) + process_args, expected_configuration.processArgs) - def initialize(self): + def initialize(self, no_create=False): """ Initialize a cluster with the configuration required by the test specification. """ + + if no_create: + try: + # If --no-create was specified and the cluster exists, skip + # initialization. If the cluster does not exist, continue + # with normal creation. + self.cluster_url.get().data + self.verify_cluster_configuration_matches(self.spec.initialConfiguration) + return + except AtlasApiError as exc: + if exc.error_code != 'CLUSTER_NOT_FOUND': + LOGGER.warn('Cluster was not found, will create one') + except AssertionError as exc: + LOGGER.warn('Configuration did not match: %s. Recreating the cluster' % exc) + LOGGER.info("Initializing cluster {!r}".format(self.cluster_name)) - cluster_config = self.spec.maintenancePlan.initial.\ + cluster_config = self.spec.initialConfiguration.\ clusterConfiguration.copy() cluster_config["name"] = self.cluster_name try: @@ -130,7 +135,7 @@ def initialize(self): raise # Apply processArgs if provided. - process_args = self.spec.maintenancePlan.initial.processArgs + process_args = self.spec.initialConfiguration.processArgs if process_args: self.client.groups[self.project.id].\ clusters[self.cluster_name].processArgs.patch(**process_args) @@ -139,23 +144,13 @@ def run(self, persist_cluster=False, startup_time=1): LOGGER.info("Running test {!r} on cluster {!r}".format( self.id, self.cluster_name)) - # Step-0: sanity-check the cluster configuration. - self.verify_cluster_configuration_matches("initial") + # Step-1: sanity-check the cluster configuration. + self.verify_cluster_configuration_matches(self.spec.initialConfiguration) # Start the test timer. timer = Timer() timer.start() - # Step-1: load test data. - test_data = self.spec.driverWorkload.get('testData') - if test_data: - LOGGER.info("Loading test data on cluster {!r}".format( - self.cluster_name)) - connection_string = self.get_connection_string() - load_test_data(connection_string, self.spec.driverWorkload) - LOGGER.info("Successfully loaded test data on cluster {!r}".format( - self.cluster_name)) - # Step-2: run driver workload. self.workload_runner.spawn( workload_executor=self.config.workload_executor, @@ -163,35 +158,80 @@ def run(self, persist_cluster=False, startup_time=1): driver_workload=self.spec.driverWorkload, startup_time=startup_time) - # Step-3: begin maintenance routine. - final_config = self.spec.maintenancePlan.final - cluster_config = final_config.clusterConfiguration - process_args = final_config.processArgs - - if not cluster_config and not process_args: - raise RuntimeError("invalid maintenance plan") - - if cluster_config: - LOGGER.info("Pushing cluster configuration update") - self.cluster_url.patch(**cluster_config) - - if process_args: - LOGGER.info("Pushing process arguments update") - self.cluster_url.processArgs.patch(**process_args) - - # Sleep before polling to give Atlas time to update cluster.stateName. - sleep(3) - - # Step-4: wait until maintenance completes (cluster is IDLE). - selector = BooleanCallablePoller( - frequency=self.config.polling_frequency, - timeout=self.config.polling_timeout) - LOGGER.info("Waiting for cluster maintenance to complete") - selector.poll([self], attribute="is_cluster_state", args=("IDLE",), - kwargs={}) - self.verify_cluster_configuration_matches("final") - LOGGER.info("Cluster maintenance complete") + for operation in self.spec.operations: + if len(operation) != 1: + raise ValueError("Operation must have exactly one key: %s" % operation) + + op_name, op_spec = list(operation.items())[0] + + if op_name == 'setClusterConfiguration': + # Step-3: begin maintenance routine. + final_config = op_spec + cluster_config = final_config.clusterConfiguration + process_args = final_config.processArgs + + if not cluster_config and not process_args: + raise RuntimeError("invalid maintenance plan") + + if cluster_config: + LOGGER.info("Pushing cluster configuration update") + self.cluster_url.patch(**cluster_config) + + if process_args: + LOGGER.info("Pushing process arguments update") + self.cluster_url.processArgs.patch(**process_args) + + # Step-4: wait until maintenance completes (cluster is IDLE). + self.wait_for_idle() + self.verify_cluster_configuration_matches(final_config) + LOGGER.info("Cluster maintenance complete") + + elif op_name == 'testFailover': + self.cluster_url['restartPrimaries'].post() + + self.wait_for_idle() + + elif op_name == 'sleep': + _time.sleep(op_spec) + + elif op_name == 'waitForIdle': + self.wait_for_idle() + + elif op_name == 'restartVms': + rv = self.admin_client.nds.groups[self.project.id].clusters[self.cluster_name].reboot.post(api_version='private') + + self.wait_for_idle() + + elif op_name == 'assertPrimaryRegion': + region = op_spec['region'] + + cluster_config = self.cluster_url.get().data + timer = Timer() + timer.start() + timeout = op_spec.get('timeout', 90) + + with mongo_client(self.get_connection_string()) as mc: + while True: + rsc = mc.admin.command('replSetGetConfig') + member = [m for m in rsc['config']['members'] + if m['horizons']['PUBLIC'] == '%s:%s' % mc.primary][0] + member_region = member['tags']['region'] + + if region == member_region: + break + + if timer.elapsed > timeout: + raise Exception("Primary in cluster not in expected region '%s' (actual region '%s')" % (region, member_region)) + else: + sleep(5) + + else: + raise Exception('Unrecognized operation %s' % op_name) + # Wait 10 seconds to ensure that the driver is not experiencing any + # errors after the maintenance has concluded. + sleep(10) + # Step-5: interrupt driver workload and capture streams stats = self.workload_runner.terminate() @@ -214,6 +254,9 @@ def run(self, persist_cluster=False, startup_time=1): # is only visible for failed tests. LOGGER.info("Workload Statistics: {}".format(stats)) + + get_logs(admin_client=self.admin_client, + project=self.project, cluster_name=self.cluster_name) # Step 7: download logs asynchronously and delete cluster. # TODO: https://github.com/mongodb-labs/drivers-atlas-testing/issues/4 @@ -223,24 +266,53 @@ def run(self, persist_cluster=False, startup_time=1): self.cluster_name)) return junit_test + + def wait_for_idle(self): + # Small delay to account for Atlas not updating cluster state + # synchronously potentially in all maintenance operations + # (https://jira.mongodb.org/browse/PRODTRIAGE-1232). + # VM restarts in sharded clusters require a much longer wait + # (30+ seconds in some circumstances); scenarios that perform + # VM restarts in sharded clusters should use explicit sleep operations + # after the restarts until this is fixed. + LOGGER.info("Waiting to wait for cluster %s to become idle" % self.cluster_name) + sleep(5) + LOGGER.info("Waiting for cluster %s to become idle" % self.cluster_name) + timer = Timer() + timer.start() + ok = False + timeout = self.config.polling_timeout + wanted_state = 'idle' + while timer.elapsed < timeout: + cluster_info = self.cluster_url.get().data + actual_state = cluster_info.stateName.lower() + if actual_state == wanted_state: + ok = True + break + LOGGER.info("Cluster %s: current state: %s; wanted state: %s; waited for %.1f sec" % (self.cluster_name, actual_state, wanted_state, timer.elapsed)) + sleep(1.0 / self.config.polling_frequency) + if not ok: + raise PollingTimeoutError("Polling timed out after %s seconds" % timeout) class SpecTestRunnerBase: """Base class for spec test runners.""" - def __init__(self, *, client, test_locator_token, configuration, xunit_output, - persist_clusters, workload_startup_time): + def __init__(self, *, client, admin_client, test_locator_token, configuration, xunit_output, + persist_clusters, no_create, workload_startup_time): self.cases = [] self.client = client + self.admin_client = admin_client self.config = configuration self.xunit_logger = SingleTestXUnitLogger(output_directory=xunit_output) self.persist_clusters = persist_clusters + self.no_create = no_create self.workload_startup_time = workload_startup_time for full_path in self.find_spec_tests(test_locator_token): # Step-1: load test specification. with open(full_path, 'r') as spec_file: test_spec = JSONObject.from_dict( - yaml.load(spec_file, Loader=yaml.FullLoader)) + yaml.safe_load(spec_file)) # Step-2: generate test name. test_name = get_test_name_from_spec_file(full_path) @@ -249,7 +321,7 @@ def __init__(self, *, client, test_locator_token, configuration, xunit_output, cluster_name = get_cluster_name(test_name, self.config.name_salt) self.cases.append( - AtlasTestCase(client=self.client, + AtlasTestCase(client=self.client, admin_client=self.admin_client, test_name=test_name, cluster_name=cluster_name, specification=test_spec, @@ -309,20 +381,15 @@ def run(self): # Step-1: initialize tests clusters for case in self.cases: - case.initialize() + case.initialize(no_create=self.no_create) # Step-2: run tests round-robin until all have been run. remaining_test_cases = self.cases.copy() while remaining_test_cases: - selector = BooleanCallablePoller( - frequency=self.config.polling_frequency, - timeout=self.config.polling_timeout) + active_case = remaining_test_cases[0] # Select a case whose cluster is ready. - LOGGER.info("Waiting for a test cluster to become ready") - active_case = selector.poll( - remaining_test_cases, attribute="is_cluster_state", - args=("IDLE",), kwargs={}) + active_case.wait_for_idle() LOGGER.info("Test cluster {!r} is ready".format( active_case.cluster_name)) diff --git a/astrolabe/utils.py b/astrolabe/utils.py index cc017bbd..f1cc7f3c 100644 --- a/astrolabe/utils.py +++ b/astrolabe/utils.py @@ -18,7 +18,9 @@ import signal import subprocess import sys +import re from hashlib import sha256 +from contextlib import closing from time import monotonic, sleep import click @@ -83,13 +85,21 @@ def create_click_option(option_spec, **kwargs): def assert_subset(dict1, dict2): """Utility that asserts that `dict2` is a subset of `dict1`, while accounting for nested fields.""" - for key, value in dict2.items(): + for key, value2 in dict2.items(): if key not in dict1: - raise AssertionError("not a subset") - if isinstance(value, dict): - assert_subset(dict1[key], value) + raise AssertionError("not a subset: '%s' from %s is not in %s" % (key, repr(dict2), repr(dict1))) + value1 = dict1[key] + if isinstance(value2, dict): + assert_subset(value1, value2) + elif isinstance(value2, list): + assert len(value1) == len(value2) + for i in range(len(value2)): + if isinstance(value2[i], dict): + assert_subset(value1[i], value2[i]) + else: + assert value1[i] == value2[i] else: - assert dict1[key] == value + assert value1 == value2, "Different values for '%s':\nexpected '%s'\nactual '%s'" % (key, repr(dict2[key]), repr(dict1[key])) class Timer: @@ -159,8 +169,7 @@ def get_cluster_name(test_name, name_salt): return name_hash.hexdigest()[:10] -def load_test_data(connection_string, driver_workload): - """Insert the test data into the cluster.""" +def mongo_client(connection_string): kwargs = {'w': "majority"} # TODO: remove this if...else block after BUILD-10841 is done. @@ -169,12 +178,8 @@ def load_test_data(connection_string, driver_workload): import certifi kwargs['tlsCAFile'] = certifi.where() client = MongoClient(connection_string, **kwargs) - - coll = client.get_database( - driver_workload.database).get_collection( - driver_workload.collection) - coll.drop() - coll.insert_many(driver_workload.testData) + + return closing(client) class DriverWorkloadSubprocessRunner: @@ -246,7 +251,9 @@ def terminate(self): else: os.kill(self.workload_subprocess.pid, signal.CTRL_BREAK_EVENT) - t_wait = 10 + # Since the default server selection timeout is 30 seconds, + # allow up to 60 seconds for the workload executor to terminate. + t_wait = 60 try: self.workload_subprocess.wait(timeout=t_wait) LOGGER.info("Stopped workload executor [PID: {}]".format(self.pid)) @@ -272,3 +279,47 @@ def terminate(self): stats = self._PLACEHOLDER_EXECUTION_STATISTICS return stats + + +def get_logs(admin_client, project, cluster_name): + data = admin_client.nds.groups[project.id].clusters[cluster_name].get(api_version='private').data + + if data['clusterType'] == 'SHARDED': + rtype = 'CLUSTER' + rname = data['deploymentItemName'] + else: + rtype = 'REPLICASET' + rname = data['deploymentItemName'] + + params = dict( + resourceName=rname, + resourceType=rtype, + redacted=True, + logTypes=['FTDC','MONGODB'],#,'AUTOMATION_AGENT','MONITORING_AGENT','BACKUP_AGENT'], + sizeRequestedPerFileBytes=100000000, + ) + data = admin_client.groups[project.id].logCollectionJobs.post(**params).data + job_id = data['id'] + + while True: + LOGGER.debug('Poll job %s' % job_id) + data = admin_client.groups[project.id].logCollectionJobs[job_id].get().data + if data['status'] == 'IN_PROGRESS': + sleep(1) + elif data['status'] == 'SUCCESS': + break + else: + raise Exception("Unexpected log collection job status %s" % data['status']) + + LOGGER.info('Log download URL: %s' % data['downloadUrl']) + # Assume the URL uses the same host as the other API requests, and + # remove it so that we just have the path. + url = re.sub(r'\w+://[^/]+', '', data['downloadUrl']) + if url.startswith('/api'): + url = url[4:] + LOGGER.info('Retrieving %s' % url) + resp = admin_client.request('GET', url) + if resp.status_code != 200: + raise RuntimeError('Request to %s failed: %s' % url, resp.status_code) + with open('logs.tar.gz', 'wb') as f: + f.write(resp.response.content) diff --git a/astrolabe/validator.py b/astrolabe/validator.py index ebb327ac..753357e8 100644 --- a/astrolabe/validator.py +++ b/astrolabe/validator.py @@ -12,24 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os, os.path from copy import deepcopy from subprocess import TimeoutExpired from time import sleep from unittest import TestCase from pymongo import MongoClient +import yaml from atlasclient import JSONObject from astrolabe.exceptions import WorkloadExecutorError -from astrolabe.utils import DriverWorkloadSubprocessRunner, load_test_data - - -DRIVER_WORKLOAD = JSONObject.from_dict({ - 'database': 'validation_db', - 'collection': 'validation_coll', - 'testData': [{'_id': 'validation_sentinel', 'count': 0}], - 'operations': [] -}) +from astrolabe.utils import DriverWorkloadSubprocessRunner class ValidateWorkloadExecutor(TestCase): @@ -39,12 +33,26 @@ class ValidateWorkloadExecutor(TestCase): def setUp(self): self.client = MongoClient(self.CONNECTION_STRING, w='majority') - self.coll = self.client.get_database( - DRIVER_WORKLOAD['database']).get_collection( - DRIVER_WORKLOAD['collection']) - load_test_data(self.CONNECTION_STRING, DRIVER_WORKLOAD) def run_test(self, driver_workload): + # Set self.coll for future use of the validator, such that it can + # read the data inserted into the collection. + # Actual insertion of initial data isn't done via this object. + dbname = None + collname = None + for e in driver_workload['createEntities']: + if dbname and collname: + break + if dbname is None and 'database' in e: + dbname = e['database']['databaseName'] + elif collname is None and 'collection' in e: + collname = e['collection']['collectionName'] + + if not (dbname and collname): + self.fail('Invalid scenario: executor validator test cases must provide database and collection entities') + + self.coll = self.client.get_database(dbname).get_collection(collname) + subprocess = DriverWorkloadSubprocessRunner() try: subprocess.spawn(workload_executor=self.WORKLOAD_EXECUTOR, @@ -79,15 +87,12 @@ def run_test(self, driver_workload): return stats def test_simple(self): - operations = [ - {'object': 'collection', - 'name': 'updateOne', - 'arguments': { - 'filter': {'_id': 'validation_sentinel'}, - 'update': {'$inc': {'count': 1}}}}] - driver_workload = deepcopy(DRIVER_WORKLOAD) - driver_workload['operations'] = operations - driver_workload = JSONObject.from_dict(driver_workload) + driver_workload = JSONObject.from_dict( + yaml.safe_load(open('tests/validator-simple.yml').read())['driverWorkload'] + ) + + if os.path.exists('events.json'): + os.unlink('events.json') stats = self.run_test(driver_workload) @@ -100,24 +105,53 @@ def test_simple(self): "statistics. Expected {} successful " "updates to be reported, got {} instead.".format( update_count, num_reported_updates)) + if abs(stats['numIterations'] - update_count) > 1: + self.fail( + "The workload executor reported inconsistent execution " + "statistics. Expected {} iterations " + "to be reported, got {} instead.".format( + update_count, stats['numIterations'])) if update_count == 0: self.fail( "The workload executor didn't execute any operations " "or didn't execute them appropriately.") + + _events = yaml.safe_load(open('events.json').read()) + if 'events' not in _events: + self.fail( + "The workload executor didn't record events as expected.") + events = _events['events'] + connection_events = [event for event in events + if event['name'].startswith('Connection')] + if not connection_events: + self.fail( + "The workload executor didn't record connection events as expected.") + pool_events = [event for event in events + if event['name'].startswith('Pool')] + if not pool_events: + self.fail( + "The workload executor didn't record connection pool events as expected.") + command_events = [event for event in events + if event['name'].startswith('Command')] + if not command_events: + self.fail( + "The workload executor didn't record command events as expected.") + for event_list in [connection_events, pool_events, command_events]: + for event in event_list: + if 'name' not in event: + self.fail( + "The workload executor didn't record event name as expected.") + if not event['name'].endswith('Event'): + self.fail( + "The workload executor didn't record event name as expected.") + if 'observedAt' not in event: + self.fail( + "The workload executor didn't record observation time as expected.") def test_num_errors(self): - operations = [ - {'object': 'collection', - 'name': 'updateOne', - 'arguments': { - 'filter': {'_id': 'validation_sentinel'}, - 'update': {'$inc': {'count': 1}}}}, - {'object': 'collection', - 'name': 'doesNotExist', - 'arguments': {'foo': 'bar'}}] - driver_workload = deepcopy(DRIVER_WORKLOAD) - driver_workload['operations'] = operations - driver_workload = JSONObject.from_dict(driver_workload) + driver_workload = JSONObject.from_dict( + yaml.safe_load(open('tests/validator-numErrors.yml').read())['driverWorkload'] + ) stats = self.run_test(driver_workload) @@ -138,6 +172,65 @@ def test_num_errors(self): "statistics. Expected approximately {} errored operations " "to be reported, got {} instead.".format( num_reported_updates, num_reported_errors)) + if abs(stats['numIterations'] - update_count) > 1: + self.fail( + "The workload executor reported inconsistent execution " + "statistics. Expected {} iterations " + "to be reported, got {} instead.".format( + update_count, stats['numIterations'])) + + def test_num_failures(self): + driver_workload = JSONObject.from_dict( + yaml.safe_load(open('tests/validator-numFailures.yml').read())['driverWorkload'] + ) + + stats = self.run_test(driver_workload) + + num_reported_finds = stats['numSuccesses'] + + num_reported_failures = stats['numFailures'] + if abs(num_reported_failures - num_reported_finds) > 1: + self.fail( + "The workload executor reported inconsistent execution " + "statistics. Expected approximately {} errored operations " + "to be reported, got {} instead.".format( + num_reported_finds, num_reported_failures)) + if abs(stats['numIterations'] - num_reported_finds) > 1: + self.fail( + "The workload executor reported inconsistent execution " + "statistics. Expected {} iterations " + "to be reported, got {} instead.".format( + num_reported_finds, stats['numIterations'])) + + def test_num_failures_as_errors(self): + driver_workload = JSONObject.from_dict( + yaml.safe_load(open('tests/validator-numFailures-as-errors.yml').read())['driverWorkload'] + ) + + stats = self.run_test(driver_workload) + + num_reported_finds = stats['numSuccesses'] + + num_reported_errors = stats['numErrors'] + num_reported_failures = stats['numFailures'] + if abs(num_reported_errors - num_reported_finds) > 1: + self.fail( + "The workload executor reported inconsistent execution " + "statistics. Expected approximately {} errored operations " + "to be reported, got {} instead.".format( + num_reported_finds, num_reported_failures)) + if num_reported_failures > 0: + self.fail( + "The workload executor reported unexpected execution " + "statistics. Expected all failures to be reported as errors, " + "got {} failures instead.".format( + num_reported_failures)) + if abs(stats['numIterations'] - num_reported_finds) > 1: + self.fail( + "The workload executor reported inconsistent execution " + "statistics. Expected {} iterations " + "to be reported, got {} instead.".format( + num_reported_finds, stats['numIterations'])) def validator_factory(workload_executor, connection_string, startup_time): diff --git a/atlasclient/client.py b/atlasclient/client.py index d9709638..8b4e4406 100644 --- a/atlasclient/client.py +++ b/atlasclient/client.py @@ -77,6 +77,7 @@ def get_path(self): class _ApiResponse: """Private wrapper class for processing HTTP responses.""" def __init__(self, response, request_method, json_data): + self.response = response self.resource_url = response.url self.headers = response.headers self.status_code = response.status_code @@ -152,6 +153,7 @@ def __init__(self, *, username, password, - `timeout` (float, optional): time, in seconds, after which an HTTP request to the Atlas API should timeout. Default: 10.0. """ + self.username=username self.config = ClientConfiguration( base_url=base_url, api_version=api_version, timeout=timeout, auth=requests.auth.HTTPDigestAuth( @@ -220,8 +222,13 @@ def request(self, method, path, **params): def construct_resource_url(self, path, api_version=None): url_template = "{base_url}/{version}/{resource_path}" + if path and path[0] == '/': + url_template = '{base_url}{resource_path}' + base_url = self.config.base_url + # Allow trailing slash like https://cloud-dev.mongodb.com/ in the base URL + base_url = base_url.rstrip('/') return url_template.format( - base_url=self.config.base_url, + base_url=base_url, version=api_version or self.config.api_version, resource_path=path) @@ -241,14 +248,18 @@ def handle_response(method, response): raise AtlasRateLimitError('Too many requests', response=response, request_method=method, error_code=429) - if data is None: + if data is None and False: raise AtlasApiError('Unable to decode JSON response.', response=response, request_method=method) kwargs = { 'response': response, 'request_method': method, - 'error_code': data.get('errorCode')} + } + + if data is not None: + kwargs['detail'] = data.get('detail') + kwargs['error_code'] = data.get('errorCode') if response.status_code == 400: raise AtlasApiError('400: Bad Request.', **kwargs) diff --git a/atlasclient/configuration.py b/atlasclient/configuration.py index 20de46b3..7b0fb377 100644 --- a/atlasclient/configuration.py +++ b/atlasclient/configuration.py @@ -27,5 +27,5 @@ # Default configuration values. CONFIG_DEFAULTS = JSONObject.from_dict({ "ATLAS_HTTP_TIMEOUT": 10.0, - "ATLAS_API_VERSION": "v1.0", - "ATLAS_API_BASE_URL": "https://cloud.mongodb.com/api/atlas"}) + "ATLAS_API_VERSION": "atlas/v1.0", + "ATLAS_API_BASE_URL": "https://cloud.mongodb.com/api"}) diff --git a/atlasclient/exceptions.py b/atlasclient/exceptions.py index e6b9cd3e..cac93d0f 100644 --- a/atlasclient/exceptions.py +++ b/atlasclient/exceptions.py @@ -17,7 +17,7 @@ class AtlasApiBaseError(Exception): """Base Exception class for all ``atlasclient`` errors.""" - def __init__(self, msg, resource_url=None, request_method=None, + def __init__(self, msg, resource_url=None, request_method=None, detail=None, status_code=None, error_code=None, headers=None): self._msg = msg self.request_method = request_method @@ -25,12 +25,13 @@ def __init__(self, msg, resource_url=None, request_method=None, self.status_code = status_code self.error_code = error_code self.headers = headers + self.detail = detail def __str__(self): if self.request_method and self.resource_url: if self.error_code: - return '{} Error Code: {!r} ({} {})'.format( - self._msg, self.error_code, self.request_method, + return '{} Error Code: {!r}: {} ({} {})'.format( + self._msg, self.error_code, self.detail, self.request_method, self.resource_url) else: return '{} ({} {})'.format( @@ -43,9 +44,9 @@ class AtlasClientError(AtlasApiBaseError): class AtlasApiError(AtlasApiBaseError): - def __init__(self, msg, response=None, request_method=None, + def __init__(self, msg, response=None, request_method=None, detail=None, error_code=None): - kwargs = {'request_method': request_method, + kwargs = {'request_method': request_method, 'detail': detail, 'error_code': error_code} # Parse remaining fields from response object. diff --git a/atlasclient/utils.py b/atlasclient/utils.py index ca004ce3..d5de3cdf 100644 --- a/atlasclient/utils.py +++ b/atlasclient/utils.py @@ -23,7 +23,7 @@ def __getattr__(self, name): if name in self: return self[name] raise AttributeError('{} has no property named {}.'.format( - self.__class__.__name__, name)) + self, name)) @classmethod def from_dict(cls, raw_dict): diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 2574e7c8..22877c6d 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -11,15 +11,19 @@ not provided an API key, or that the API key that you have provided is has expir see the `MongoDB Atlas API `_ documentation for instructions on how to create programmatic API keys. +You also need a set of API keys with Atlas global operator permissions, +referred to as admin credentials. + ``astrolabe`` can be configured to use API keys in one of 2 ways: * Using the `-u/--username` and `-p/--password` command options:: - $ astrolabe -u -p check-connection + $ astrolabe -u -p --atlas-admin-api-username --atlas-admin-api-password check-connection -* Using the ``ATLAS_API_USERNAME`` and ``ATLAS_API_PASSWORD`` environment variables:: +* Using the ``ATLAS_API_USERNAME``, ``ATLAS_API_PASSWORD``, + ``ATLAS_ADMIN_API_USERNAME``, ``ATLAS_ADMIN_API_PASSWORD`` environment variables:: - $ ATLAS_API_USERNAME= ATLAS_API_PASSWORD= astrolabe check-connection + $ ATLAS_API_USERNAME= ATLAS_API_PASSWORD= ATLAS_ADMIN_API_USERNAME= ATLAS_ADMIN_API_PASSWORD= astrolabe check-connection .. _faq-why-custom-distro: diff --git a/docs/source/installing-running-locally.rst b/docs/source/installing-running-locally.rst index 254ca81a..386ac934 100644 --- a/docs/source/installing-running-locally.rst +++ b/docs/source/installing-running-locally.rst @@ -43,14 +43,37 @@ Before you can start using ``astrolabe``, you must configure it to give it acces If you haven't done so already, create a `MongoDB Atlas Organization `_ (this can -only be done via the Atlas UI). Make a note of the name of the Atlas organization. You will also need -a `Programmatic API Key ` for this Atlas Organization with -"Organization Owner" permissions. The API key will consist of 2 parts - a public key and a private key. -Finally, declare the following variables to configure ``astrolabe``:: +only be done via the Atlas UI). Make a note of the name of the Atlas organization. + +Depending on the test scenario being executed, you will need either one +or two sets of a `Programmatic API Keys +`_: a regular +key for Atlas Organization you created with "Organization Owner" permissions, +and potentially a key with Atlas Global Operator permissions (hereafter +referred to as the "admin key"). The admin key generally must be created by +a Cloud team member and would typically be issued for the development environment +of Atlas (`https://cloud-dev.mongodb.com `_), +meaning the organization and projects must also be created in the development +environment. + +Each API key consists of 2 parts - a public key and a private key. + +To configure ``astrolabe`` to use production Atlas and specify only a regular +API key, declare the following variables:: + + $ export ATLAS_ORGANIZATION_NAME= + $ export ATLAS_API_USERNAME= + $ export ATLAS_API_PASSWORD= + +To configure ``astrolabe`` to use development Atlas and specify two sets of +API keys, declare the following variables:: $ export ATLAS_ORGANIZATION_NAME= $ export ATLAS_API_USERNAME= $ export ATLAS_API_PASSWORD= + $ export ATLAS_ADMIN_API_USERNAME= + $ export ATLAS_ADMIN_API_PASSWORD= + $ export ATLAS_API_BASE_URL=https://cloud-dev.mongodb.com/api/atlas Finally, use the ``check-connection`` command to confirm that ``astrolabe`` is able to connect to and authenticate with the Atlas API:: @@ -87,11 +110,11 @@ Running Atlas Planned Maintenance Tests The ``spec-tests`` command-group is used for Atlas Planned Maintenance (APM) tests. To run a single APM test, do:: - $ astrolabe spec-tests run-one -e --project-name --cluster-name-salt + $ astrolabe spec-tests run-one -e --project-name --cluster-name-salt where: -* ```` is the absolute or relative path to a test scenario file in the +* ```` is the absolute or relative path to a test scenario file in the :ref:`test-scenario-format-specification`, * ```` is the absolute or relative path to the workload executor of the driver to be tested, * ```` is the name of the Atlas Project under which the test cluster used for the test will be created, @@ -118,6 +141,15 @@ Using this flag with a given test file and static ``--cluster-name-salt`` value times between successive test runs (you will still need to wait for the cluster to be reconfigured to the initial configuration). +``astrolabe`` also provides the ``--no-create`` flag which makes it skip +cluster initialization if the cluster already exists. This flag may be used +to further speed up the test runs, but it can only be used for scenarios +where the cluster configuration does not change from the initial one +(otherwise the test would start with the wrong configuration). Using +``--no-delete`` is recommended with ``--no-create``, otherwise each run will +delete the cluster upon completion. + + Debugging --------- diff --git a/docs/source/integration-guide.rst b/docs/source/integration-guide.rst index f6219a77..e647f8c1 100644 --- a/docs/source/integration-guide.rst +++ b/docs/source/integration-guide.rst @@ -272,3 +272,20 @@ in the Evergreen configuration file: .. note:: Users are asked to be extra cautious while dealing with environment variables that contain sensitive secrets. Using these variables in a script that sets ``-xtrace`` can, for instance, result in leaking these secrets into Evergreen's log output. + +--------------- +Troubleshooting +--------------- + +When using ``cloud-dev``, be aware that operational issues within Atlas are +not being monitored and solved with a particular SLA. If builds are failing +and the failure appears to be caused by Atlas rather than the tests themselves, +the driver being tested or ``astrolabe``, inquiring in ``cloud-non-prod-ops`` +Slack channel is the next suggested troubleshooting step. + +Atlas has a limit of 40 "cross-region network permissions" by default. +This means a project can have no more than 40 nodes across all of its +clusters if any of its clusters employ multiple regions. The primary +takeover and primary removal tests use multi-region clusters; running +these tests alongside other tests may exceed the 40 node limit. A +request to the Cloud team is required to raise the limit. diff --git a/docs/source/spec-test-format.rst b/docs/source/spec-test-format.rst index fd11fb36..3c0ff349 100644 --- a/docs/source/spec-test-format.rst +++ b/docs/source/spec-test-format.rst @@ -10,19 +10,23 @@ The YAML file format described herein is used to define platform-independent *At YAML-formatted *Test Scenario Files*. Each Test Scenario File describes exactly one Atlas Planned Maintenance Test. A Test Scenario File has the following keys: -* maintenancePlan (document): a *Planned Maintenance Scenario* object. Each object has the following keys: +* initialConfiguration (document): Description of *Cluster Configuration Options* to be used for initializing the + test cluster. This document MUST contain the following keys: - * initialConfiguration (document): Description of *Cluster Configuration Options* to be used for initializing the - test cluster. This document MUST contain the following keys: + * clusterConfiguration (document): Document containing initial *Basic Configuration Options* values. + This document MUST, at a minimum, have all fields **required** by the + `Create One Cluster `_ endpoint. + * processArgs (document): Document containing initial *Advanced Configuration Option* values. This MAY be an empty + document if the maintenance plan does not require modifying the Advanced Configuration Options. - * clusterConfiguration (document): Document containing initial *Basic Configuration Options* values. - This document MUST, at a minimum, have all fields **required** by the - `Create One Cluster `_ endpoint. - * processArgs (document): Document containing initial *Advanced Configuration Option* values. This MAY be an empty - document if the maintenance plan does not require modifying the Advanced Configuration Options. - - * finalConfiguration (document): Description of **new** *Cluster Configuration Options* to be applied to the - test cluster. This document MUST contain the following keys (note that at least one of these fields MUST be +* operations (array): List of operations to be performed, representing the + maintenance event. Each operation is a document containing one key which is + the name of the operation. The possible operations are: + + * setClusterConfiguration: set the cluster configuration to the specified + *Cluster Configuration Options* as defined in initialConfiguration. + The value must be the *Cluster Configuration Options* which MUST contain + the following keys (note that at least one of these fields MUST be a non-empty document): * clusterConfiguration (document): Document containing final *Basic Configuration Option* values. @@ -31,25 +35,124 @@ A Test Scenario File has the following keys: `Modify One Cluster `_ endpoint. * processArgs (document): Document containing final *Advanced Configuration Option* values. This MAY be an empty document if the maintenance plan does not require modifying the Advanced Configuration Options. + + Example:: + + setClusterConfiguration: + clusterConfiguration: + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + + * testFailover: trigger an election in the cluster using the "test failover" + API endpoint. The value MUST be ``true``. + + The workload executor MUST ignore the value of this key, so that + the value can be changed to a hash in the future to provide options + to the operation. + + testFailover SHOULD be followed by sleep and waitForIdle operations + because it does not update maintenance state synchronously (see + `PRODTRIAGE-1232 `_). + + Example:: + + testFailover: true + + * restartVms: perform a rolling restart of all nodes in the cluster. + This operation requires Atlas Global Operator API key to be set when + invoking ``astrolabe``. The value MUST be ``true``. + + The workload executor MUST ignore the value of this key, so that + the value can be changed to a hash in the future to provide options + to the operation. + + testFailover SHOULD be followed by sleep and waitForIdle operations + because it does not update maintenance state synchronously. + + Example:: + + restartVms: true + + * assertPrimaryRegion: assert that the primary in the deployment is in the + specified region. The value MUST be a hash with the following keys: + + * region (string, required): the region name as defined in Atlas API, + e.g. ``US_WEST_1``. + * timeout (floating-point number, optional): the maximum time, in + seconds, to wait for the region to become the expected one. + Default is 90 seconds. + + This operation is undefined and MUST NOT be used when the deployment is + a sharded cluster. + + Example:: + + assertPrimaryRegion: + region: US_WEST_1 + timeout: 15 + + * sleep: do nothing for the specified duration. The value MUST be the duration + to sleep for, in seconds. + + Example:: + + sleep: 10 + + * waitForIdle: wait for cluster maintenance state to become "idle". + The value MUST be ``true``. + + The workload executor MUST ignore the value of this key, so that + the value can be changed to a hash in the future to provide options + to the operation. + + Example:: + + waitForIdle: true - * uriOptions (document): Document containing ``key: value`` pairs of URI options that must be included in the - connection string passed to the workload executor by the *Test Orchestrator*. + For all maintenance operations other than ``sleep``, after the maintenance + operation is performed, ``astrolabe`` will wait for cluster state to become + idle. When performing a VM restart in a sharded cluster, due to the state + not being updated for a potentially long time, the test SHOULD add an + explicit ``sleep`` operation for at least 30 seconds. -* driverWorkload (document): Object describing a *Driver Workload*. Has the following keys: +* driverWorkload (document): Description of the driver workload to execute + The document must be a complete test as defined by the + `Unified Test Format specification `_. + + The workload MUST use a single test, as defined in the unified test format + specification. + + The workload MUST use the ``loop`` unified test format operation to + define the MongoDB operations to execute during maintenance. There MUST + be exactly one ``loop`` operation per scenario, and it SHOULD be the last + operation in the scenario. - * collection (string): Name of the collection to use for running test operations. - * database (string): Name of the database to use for running test operations. - * testData (array, optional): Array of documents to be inserted into the ``database.collection`` namespace before - starting the test run. Test data insertion is performed by the *Test Orchestrator* and this field MUST be ignored - by the *Workload Executor*. - * operations (array): Array of Operation objects, each describing an operation to be executed. The operations are run - sequentially and repeatedly until the maintenance completes. Each object has the following keys: + The scenario MUST use ``storeErrorsAsEntity``, ``storeFailuresAsEntity``, + ``storeSuccesesAsEntity`` and ``storeIterationsAsEntity`` operation arguments + to allow the workload executor to retrieve errors, failures and operation + counts for the executed workload. The entity names for these options MUST + be as follows: + + - ``storeErrorsAsEntity``: ``errors`` + - ``storeFailuresAsEntity``: ``failures`` + - ``storeSuccessesAsEntity``: ``successes`` + - ``storeIterationsAsEntity``: ``iterations`` + + The scenario MUST use ``storeEventsAsEntities`` operation argument + when defining MongoClients to record CMAP and command events published + during maintenance. The entity name for ``storeEventsAsEntities`` argument + MUST be ``events``. When this option is used, ``astrolabe`` will retrieve + the collected events and store them as an Evergreen build artifact, and + will also calculate statistics for command execution time and connection + counts. - * object (string): The entity on which to perform the operation. Can be "database" or "collection". - * name (string): name of the operation. - * arguments (document): the names and values of arguments to be passed to the operation. - * result (optional, multiple types): The result of executing the operation. This will correspond to the operation's - return value. +.. note:: A previous version of this document specified a top-level + ``uriOptions`` for specifying URI options for the MongoClient under test. + In the current version, options can be specified using the ``uriOptions`` + key of the unified test format when creating a client entity. ------- Changes diff --git a/docs/source/spec-workload-executor.rst b/docs/source/spec-workload-executor.rst index 9c892e81..f5e4e84a 100644 --- a/docs/source/spec-workload-executor.rst +++ b/docs/source/spec-workload-executor.rst @@ -37,55 +37,133 @@ Behavioral Description After accepting the inputs, the workload executor: -#. MUST use the input connection string to instantiate the ``MongoClient`` of the driver that is to be tested. - Note that the workload executor: +#. MUST use the input connection string to `instantiate the + unified test runner `_ + of the driver being tested. Note that the workload executor: * MUST NOT override any of the URI options specified in the incoming connection string. * MUST NOT augment the incoming connection string with any additional URI options. -#. MUST parse the incoming the ``driverWorkload`` document and use the ``MongoClient`` instance from the previous step - to run the operations described therein in accordance with the :ref:`test-scenario-format-specification`. - Note that the workload executor: - - * MUST ignore the ``testData`` array. ``astrolabe`` is responsible for initializing the cluster with - this data *before* starting the workload executor. - * MUST run operations sequentially and in the order in which they appear in the ``operations`` array. - * MUST repeat the entire set of specified operations indefinitely, until the **termination signal** from - ``astrolabe`` is received. - * MUST keep count of the number of operations failures (``numFailures``) that are encountered while running - operations. An operation failure is when the actual return value of an operation does not match its - expected return value (as defined in the ``result`` field of the ``driverWorkload``). - * MUST keep count of the number of operation errors (``numErrors``) that are encountered while running - operations. An operation error is when running an operation unexpectedly raises an error. Workload executors - implementations should try to be as resilient as possible to these kinds of operation errors. - * MUST keep count of the number of operations that are run successfully (``numSuccesses``). - -#. MUST set a signal handler for handling the termination signal that is sent by ``astrolabe``. The termination signal - is used by ``astrolabe`` to communicate to the workload executor that it should stop running operations. Upon - receiving the termination signal, the workload executor: - - * MUST stop running driver operations and exit soon. - * MUST dump collected workload statistics as a JSON file named ``results.json`` in the current working directory - (i.e. the directory from where the workload executor is being executed). Workload statistics MUST contain the - following fields (drivers MAY report additional statistics using field names of their choice): - - * ``numErrors``: the number of operation errors that were encountered during the test. - * ``numFailures``: the number of operation failures that were encountered during the test. - * ``numSuccesses``: the number of operations executed successfully during the test. - - .. note:: The values of ``numErrors`` and ``numFailures`` are used by ``astrolabe`` to determine the overall - success or failure of a driver workload execution. A non-zero value for either of these fields is construed - as a sign that something went wrong while executing the workload and the test is marked as a failure. - The workload executor's exit code is **not** used for determining success/failure and is ignored. - - .. note:: If ``astrolabe`` encounters an error in parsing the workload statistics dumped to ``results.json`` - (caused, for example, by malformed JSON), ``numErrors``, ``numFailures``, and ``numSuccesses`` - will be set to ``-1`` and the test run will be assumed to have failed. - - .. note:: The choice of termination signal used by ``astrolabe`` varies by platform. ``SIGINT`` [#f1]_ is used as - the termination signal on Linux and OSX, while ``CTRL_BREAK_EVENT`` [#f2]_ is used on Windows. - - .. note:: On Windows systems, the workload executor is invoked via Cygwin Bash. +#. MUST parse the incoming ``driverWorkload`` document and set up + the driver's unified test runner to execute the provided workload. + + .. note:: + + The workload SHOULD include a ``loop`` operation, as described in the + unified test format, but the workload executor SHOULD NOT validate that + this is the case. + +#. MUST set a signal handler for handling the termination signal that is + sent by ``astrolabe``. The termination signal is used by ``astrolabe`` + to communicate to the workload executor, and ultimately the unified test + runner, that they should stop running operations. + +#. MUST invoke the unified test runner to execute the workload. + If the workload includes a ``loop`` operation, the workload will run until + terminated by the workload executor; otherwise, the workload will terminate + when the unified test runner finishes executing all of the operations. + The workload executor MUST handle the case of a non-looping workload and + it MUST terminate if the unified test runner completely executes the + specified workload. + + If the unified test runner raises an error while executing the workload, + the error MUST be reported using the same format as errors handled by the + unified test runner, as described in the unified test runner specification + under the ``loop`` operation. Errors handled by the workload + executor MUST be included in the calculated (and reported) error count. + + If the unified test runner reports a failure while executing the workload, + the failure MUST be reported using the same format as failures handled by the + unified test runner, as described in the unified test runner specification + under the ``loop`` operation. Failures handled by the workload + executor MUST be included in the calculated (and reported) failure count. + If the driver's unified test runner is intended to handle all failures + internally, failures that propagate out of the unified test runner MAY + be treated as errors by the workload executor. + +#. Upon receipt of the termination signal, MUST instruct the + unified test runner to stop looping, as defined in the unified test format. + +#. MUST wait for the unified test runner to finish executing. + +#. MUST use the unified test runner to retrieve the following + entities by name from the entity map, if they are set: + + * ``iterations``: the number of iterations that the workload executor + performed over the looped operations. If the iteration count was not + reported by the test runner, such as because the respective option was + not specified in the test scenario, the workload executor MUST use + ``-1`` as the number of iterations. + + * ``successes``: the number of successful operations that the workload + executor performed over the looped operations. If the iteration count + was not reported by the test runner, such as because the respective + option was not specified in the test scenario, the workload executor + MUST use ``-1`` as the number of successes. + + * ``errors``: array of documents describing the errors that occurred + while the workload executor was executing the operations. + + * ``failures``: array of documents describing the failures that occurred + while the workload executor was executing the operations. + + * ``events``: array of documents describing the command and CMAP events + that occurred while the workload executor was executing the operations. + + If the driver's unified test format does not distinguish between errors + and failures, and reports one but not the other, the workload executor MUST + set the non-reported entry to the empty array. + +#. MUST calculate the aggregate counts of errors (``numErrors``) and failures + (``numFailures``) from the error and failure lists. If the errors or + failures were not reported by the test runner, such as because the + respective options were not specified in the test scenario, the workload + executor MUST use ``-1`` as the value for the respective counts. + +#. MUST write the collected events, errors and failures into a JSON file named + ``events.json`` in the current directory + (i.e. the directory from where the workload executor is being executed). + The data written MUST be a map with the following fields: + + - ``events``: the collected command and CMAP events. + + - ``errors``: the reported errors. + + - ``failures``: the reported errors. + + If events, errors or failures were not reported by the unified test runner, + such as because the scenario did not specify the corresponding options, + the workload executor MUST write empty arrays into ``events.json``. + +#. MUST write the collected workload statistics into a JSON file named + ``results.json`` in the current working directory (i.e. the directory + from where the workload executor is being executed). Workload statistics + MUST contain the following fields (drivers MAY report additional statistics + using field names of their choice): + + * ``numErrors``: the number of operation errors that were encountered + during the test. This includes errors handled by the workload executor + and errors handled by the unified test runner. + * ``numFailures``: the number of operation failures that were encountered + during the test. This includes failures handled by the workload executor + and failures handled by the unified test runner. + * ``numSuccesses``: the number of successful operations executed + during the test. + * ``numIterations``: the number of loop iterations executed during the test. + + .. note:: The values of ``numErrors`` and ``numFailures`` are used by ``astrolabe`` to determine the overall + success or failure of a driver workload execution. A non-zero value for either of these fields is construed + as a sign that something went wrong while executing the workload and the test is marked as a failure. + The workload executor's exit code is **not** used for determining success/failure and is ignored. + +.. note:: If ``astrolabe`` encounters an error in parsing the workload statistics dumped to ``results.json`` + (caused, for example, by malformed JSON), ``numErrors``, ``numFailures``, and ``numSuccesses`` + will be set to ``-1`` and the test run will be assumed to have failed. + +.. note:: The choice of termination signal used by ``astrolabe`` varies by platform. ``SIGINT`` [#f1]_ is used as + the termination signal on Linux and OSX, while ``CTRL_BREAK_EVENT`` [#f2]_ is used on Windows. + +.. note:: On Windows systems, the workload executor is invoked via Cygwin Bash. Pseudocode Implementation @@ -96,54 +174,70 @@ Pseudocode Implementation # targetDriver is the driver to be tested. import { MongoClient } from "targetDriver" - # The workloadRunner function accepts a connection string and a stringified JSON blob describing the driver workload. - # This function will be invoked with arguments parsed from the command-line invocation of the workload executor script. + # The workloadRunner function accepts a connection string and a + # stringified JSON blob describing the driver workload. + # This function will be invoked with arguments parsed from the + # command-line invocation of the workload executor script. function workloadRunner(connectionString: string, driverWorkload: object): void { - # Use the MongoClient of the driver to be tested to connect to the Atlas Cluster. - const client = MongoClient(connectionString); - - # Create objects which will be used to run operations. - const db = client.db(driverWorkload.database); - const collection = db.collection(driverWorkload.collection); - - # Initialize counters. - var num_errors = 0; - var num_failures = 0; - var num_successes = 0; - - # Run the workload - operations are run sequentially, repeatedly until the termination signal is received. - # Do not attempt to initialize the cluster with the contents of ``testData`` - astrolabe takes care of this. + # Use the driver's unified test runner to run the workload. + const runner = UnifiedTestRunner(connectionString); + try { - while (True) { - for (let operation in workloadSpec.operations) { - try { - # The runOperation method runs operations as per the test format. - # The method return False if the actual return value of the operation does match the expected. - var was_succesful = runOperation(db, collection, operation); - if (was_successful) { - num_successes += 1; - } else { - num_errors += 1; - } - } catch (operationError) { - # We end up here if runOperation raises an unexpected error. - num_failures += 1; - } - } - } + runner.executeScenario(); } catch (terminationSignal) { # The workloadExecutor MUST handle the termination signal gracefully. # The termination signal will be used by astrolabe to terminate drivers operations that otherwise run ad infinitum. # The workload statistics must be written to a file named results.json in the current working directory. - fs.writeFile('results.json', JSON.stringify({‘numErrors’: num_errors, 'numFailures': num_failures, 'numSuccesses': num_successes})); } + + let results = {}; + try { + numIterations = runner.entityMap.get('iterations'); + } catch { + numIterations = -1; + } + try { + numSuccesses = runner.entityMap.get('successes'); + } catch { + numSuccesses = -1; + } + try { + errors = runner.entityMap.get('errors'); + numErrors = errors.length; + } catch { + errors = []; + numErrors = -1; + } + try { + failures = runner.entityMap.get('failures'); + numFailures = failures.length; + } catch { + failures = []; + numFailures = -1; + } + try { + events = runner.entityMap.get('events'); + } catch { + events = []; + } + + fs.writeFile('events.json', JSON.stringify({ + events: events, + errors: errors, + failures: failures, + })); + fs.writeFile('results.json', JSON.stringify({ + ‘numErrors’: numErrors, + 'numFailures': numFailures, + 'numSuccesses': numSuccesses, + })); } Reference Implementation ------------------------ -`PyMongo's workload executor `_ +`Ruby's workload executor `_ serves as the reference implementation of the script described by this specification. diff --git a/docs/source/technical-design.rst b/docs/source/technical-design.rst index cd226567..bd1dfbca 100644 --- a/docs/source/technical-design.rst +++ b/docs/source/technical-design.rst @@ -164,13 +164,13 @@ User-Facing API The Test Orchestrator MUST be an executable that supports the following invocation pattern:: - ./test-orchestrator spec-tests run-one path/to/workload-spec.yaml -e path/to/workload-executor + ./test-orchestrator spec-tests run-one path/to/workload-spec.yml -e path/to/workload-executor where: * ``test-orchestrator`` is the Test Orchestrator executable, * ``spec-tests run-one`` is the name of the command issued to this executable, -* ``path/to/workload-spec.yaml`` is the path to a test scenario file, +* ``path/to/workload-spec.yml`` is the path to a test scenario file, * ``-e`` is a flag indicating that the following argument is the workload executor binary, and * ``path/to/workload-executor`` is the path to the workload executor binary that is to be used to run the Driver Workload. diff --git a/integrations/ruby/executor.rb b/integrations/ruby/executor.rb index a809c356..4f774a47 100644 --- a/integrations/ruby/executor.rb +++ b/integrations/ruby/executor.rb @@ -1,5 +1,6 @@ require 'json' require 'mongo' +require 'runners/unified' Mongo::Logger.logger.level = Logger::WARN @@ -9,23 +10,26 @@ class UnknownOperationConfiguration < StandardError; end class Executor def initialize(uri, spec) @uri, @spec = uri, spec - @operation_count = @failure_count = @error_count = 0 + @iteration_count = @success_count = @failure_count = @error_count = 0 end attr_reader :uri, :spec - attr_reader :operation_count, :failure_count, :error_count + attr_reader :iteration_count, :failure_count, :error_count def run + unified_tests + set_signal_handler - # Normally, the orchestrator loads test data. - # If the executor is run by itself, uncomment the next line. - #load_data - while true - break if @stop - perform_operations + unified_tests.each do |test| + test.create_entities + test.set_initial_data + test.run + test.assert_outcome + test.assert_events + test.cleanup end - puts "Result: #{result.inspect}" write_result + puts "Result: #{result.inspect}" end private @@ -33,105 +37,72 @@ def run def set_signal_handler Signal.trap('INT') do @stop = true + unified_tests.each do |test| + test.stop! + end end end - def load_data - collection.delete_many - if data = spec['testData'] - collection.insert_many(data) - end + def unified_group + @unified_group ||= Unified::TestGroup.new(spec, + client_args: uri, kill_sessions: false) end - def perform_operations - spec['operations'].each do |op_spec| - begin - case op_spec['name'] - when 'find' - unless op_spec['object'] == 'collection' - raise UnknownOperationConfiguration, "Can only find on a collection" - end - - args = op_spec['arguments'].dup - op = collection.find(args.delete('filter') || {}) - if sort = args.delete('sort') - op = op.sort(sort) - end - unless args.empty? - raise UnknownOperationConfiguration, "Unhandled keys in args: #{args}" - end - - docs = op.to_a - - if expected_docs = op_spec['result'] - if expected_docs != docs - puts "Failure" - @failure_count += 1 - end - end - when 'insertOne' - unless op_spec['object'] == 'collection' - raise UnknownOperationConfiguration, "Can only find on a collection" - end - - args = op_spec['arguments'].dup - document = args.delete('document') - unless args.empty? - raise UnknownOperationConfiguration, "Unhandled keys in args: #{args}" - end - - collection.insert_one(document) - when 'updateOne' - unless op_spec['object'] == 'collection' - raise UnknownOperationConfiguration, "Can only find on a collection" - end - - args = op_spec['arguments'].dup - scope = collection - if filter = args.delete('filter') - scope = collection.find(filter) - end - if update = args.delete('update') - scope.update_one(update) - end - unless args.empty? - raise UnknownOperationConfiguration, "Unhandled keys in args: #{args}" - end - else - raise UnknownOperation, "Unhandled operation #{op_spec['name']}" - end - #rescue Mongo::Error => e - # The validator intentionally gives us invalid operations, figure out - # how to handle this requirement while maintaining diagnostics. - rescue => e - STDERR.puts "Error: #{e.class}: #{e}" - @error_count += 1 - end - @operation_count += 1 - end + def unified_tests + @tests ||= unified_group.tests end def result { - numOperations: @operation_count, - numSuccessfulOperations: @operation_count-@error_count-@failure_count, - numSuccesses: @operation_count-@error_count-@failure_count, + numIterations: @iteration_count, + numSuccessfulOperations: @success_count, + numSuccesses: @success_count, numErrors: @error_count, numFailures: @failure_count, } end def write_result + {}.tap do |event_result| + @iteration_count = -1 + @success_count = -1 + @events = [] + @errors = [] + @failures = [] + unified_tests.map do |test| + begin + @iteration_count += test.entities.get(:iteration_count, 'iterations') + rescue Unified::Error::EntityMissing + end + begin + @success_count += test.entities.get(:success_count, 'successes') + rescue Unified::Error::EntityMissing + end + begin + @events += test.entities.get(:event_list, 'events') + rescue Unified::Error::EntityMissing + end + begin + @errors += test.entities.get(:error_list, 'errors') + rescue Unified::Error::EntityMissing + end + begin + @failures += test.entities.get(:failure_list, 'failures') + rescue Unified::Error::EntityMissing + end + end + @error_count += @errors.length + @failure_count += @failures.length + File.open('events.json', 'w') do |f| + f << JSON.dump( + errors: @errors, + failures: @failures, + events: @events, + ) + end + end File.open('results.json', 'w') do |f| f << JSON.dump(result) end end - - def collection - @collection ||= client.use(spec['database'])[spec['collection']] - end - - def client - @client ||= Mongo::Client.new(uri) - end end diff --git a/integrations/ruby/workload-executor b/integrations/ruby/workload-executor index 03e79036..d4956b95 100755 --- a/integrations/ruby/workload-executor +++ b/integrations/ruby/workload-executor @@ -6,16 +6,61 @@ puts ([$0] + ARGV.map { |arg| Shellwords.shellescape(arg) }).join(' ') $: << File.dirname(__FILE__) $: << File.join(File.dirname(__FILE__), '../../mongo-ruby-driver/lib') +$: << File.join(File.dirname(__FILE__), '../../mongo-ruby-driver/spec') +$: << File.join(File.dirname(__FILE__), '../../../ruby-driver/lib') +$: << File.join(File.dirname(__FILE__), '../../../ruby-driver/spec') require 'executor' +require 'optparse' +autoload :Byebug, 'byebug' + +options = {} +OptionParser.new do |opts| + opts.banner = $usage_banner = "Usage: workload-executor URI SPEC\n" << + " workload-executor -s SCENARIO-PATH [-i] [-u URI]" + + opts.on("-s", "--scenario=PATH", "Specify scenario path") do |v| + options[:scenario_path] = v + end + opts.on('-i', '--insert', 'Insert scenario data') do + options[:insert] = true + end + opts.on("-u", "--uri=URI", "Specify MongoDB server URI") do |v| + options[:uri] = v + end +end.parse! uri, spec = ARGV +uri ||= options[:uri] +uri ||= 'mongodb://localhost' + +if spec.nil? && !options[:scenario_path] + STDERR.puts $usage_banner + exit 1 +end -if spec.nil? - raise "Usage: executor.rb URI SPEC" +if options[:scenario_path] + scenario = YAML.load(File.read(options[:scenario_path])) + spec = scenario.fetch('driverWorkload') +else + spec = JSON.load(spec) end -spec = JSON.load(spec) +$uri = uri + +class ClientRegistry + def self.instance + new + end + + def global_client(which) + $global_client ||= Mongo::Client.new($uri) + end +end executor = Executor.new(uri, spec) -executor.run +if options[:insert] + executor.load_data +else + executor.run +end diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..3a7574b2 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +sphinx diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..85aa1cd1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +click>=7,<8 +requests>=2,<3 +pymongo>=3.10,<4 +dnspython>=1.16,<2 +pyyaml>=5,<6 +tabulate>=0.8,<0.9 +junitparser>=1,<2 +numpy diff --git a/setup.py b/setup.py index f66e3035..af36d251 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ 'click>=7,<8', 'requests>=2,<3', 'pymongo>=3.10,<4', 'dnspython>=1.16,<2', 'pyyaml>=5,<6', 'tabulate>=0.8,<0.9', + 'numpy<2', 'junitparser>=1,<2'] if sys.platform == 'win32': install_requires.append('certifi') diff --git a/tests/README.rst b/tests/README.rst index e5be0781..81615aee 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -12,7 +12,7 @@ Test File Naming Convention The names of test files serve as the names of the tests themselves (as displayed in the Evergreen UI). Consequently, it is recommended that file names observe the following naming convention:: - -.yaml + -.yml Use of ``camelCase`` is recommended for specifying the driver workload and maintenance plan names. Ideally, these names should be descriptive enough to be self-explanatory though this might not be possible for more complex workloads diff --git a/tests/retryReads-move-sharded.yml b/tests/retryReads-move-sharded.yml new file mode 100644 index 00000000..cb435189 --- /dev/null +++ b/tests/retryReads-move-sharded.yml @@ -0,0 +1,84 @@ +initialConfiguration: + clusterConfiguration: + clusterType: SHARDED + providerSettings: + providerName: AWS + regionName: US_EAST_1 + instanceSizeName: M10 + processArgs: {} + +operations: + - + setClusterConfiguration: + clusterConfiguration: + providerSettings: + providerName: AWS + regionName: US_EAST_1 + instanceSizeName: M20 + processArgs: {} + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-move.yml b/tests/retryReads-move.yml new file mode 100644 index 00000000..69c250dc --- /dev/null +++ b/tests/retryReads-move.yml @@ -0,0 +1,84 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M20 + processArgs: {} + +operations: + - + setClusterConfiguration: + clusterConfiguration: + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-primaryRemoval.yml b/tests/retryReads-primaryRemoval.yml new file mode 100644 index 00000000..41949cf7 --- /dev/null +++ b/tests/retryReads-primaryRemoval.yml @@ -0,0 +1,112 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + instanceSizeName: M10 + replicationSpecs: + - + id: '111111111111111111111111' + numShards: 1 + regionsConfig: + US_WEST_1: + electableNodes: 2 + priority: 6 + readOnlyNodes: 0 + US_EAST_1: + electableNodes: 1 + priority: 7 + readOnlyNodes: 0 + processArgs: {} + +operations: + - assertPrimaryRegion: + region: US_EAST_1 + + - + setClusterConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + instanceSizeName: M10 + replicationSpecs: + - + id: '111111111111111111111111' + numShards: 1 + regionsConfig: + US_WEST_1: + electableNodes: 3 + priority: 7 + readOnlyNodes: 0 + processArgs: {} + + - assertPrimaryRegion: + region: US_WEST_1 + + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-primaryTakeover.yml b/tests/retryReads-primaryTakeover.yml new file mode 100644 index 00000000..64d0a6db --- /dev/null +++ b/tests/retryReads-primaryTakeover.yml @@ -0,0 +1,113 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + instanceSizeName: M10 + replicationSpecs: + - + id: '111111111111111111111111' + numShards: 1 + regionsConfig: + US_WEST_1: + electableNodes: 3 + priority: 7 + readOnlyNodes: 0 + processArgs: {} + +operations: + - assertPrimaryRegion: + region: US_WEST_1 + + - + setClusterConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + instanceSizeName: M10 + replicationSpecs: + - + id: '111111111111111111111111' + numShards: 1 + regionsConfig: + US_WEST_1: + electableNodes: 2 + priority: 6 + readOnlyNodes: 0 + US_EAST_1: + electableNodes: 1 + priority: 7 + readOnlyNodes: 0 + processArgs: {} + + - assertPrimaryRegion: + region: US_EAST_1 + timeout: 60 + + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-processRestart-sharded.yml b/tests/retryReads-processRestart-sharded.yml new file mode 100644 index 00000000..58e8cce4 --- /dev/null +++ b/tests/retryReads-processRestart-sharded.yml @@ -0,0 +1,86 @@ +initialConfiguration: + clusterConfiguration: + clusterType: SHARDED + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: + minimumEnabledTlsProtocol: TLS1_1 + +operations: + - + setClusterConfiguration: + clusterConfiguration: + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: + minimumEnabledTlsProtocol: TLS1_2 + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-processRestart.yml b/tests/retryReads-processRestart.yml new file mode 100644 index 00000000..96647bfd --- /dev/null +++ b/tests/retryReads-processRestart.yml @@ -0,0 +1,86 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: + minimumEnabledTlsProtocol: TLS1_1 + +operations: + - + setClusterConfiguration: + clusterConfiguration: + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: + minimumEnabledTlsProtocol: TLS1_2 + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-resizeCluster.yaml b/tests/retryReads-resizeCluster.yaml deleted file mode 100644 index 87d2805c..00000000 --- a/tests/retryReads-resizeCluster.yaml +++ /dev/null @@ -1,37 +0,0 @@ -maintenancePlan: - initial: - clusterConfiguration: - clusterType: REPLICASET - providerSettings: - providerName: AWS - regionName: US_WEST_1 - instanceSizeName: M10 - processArgs: {} - final: - clusterConfiguration: - providerSettings: - providerName: AWS - regionName: US_WEST_1 - instanceSizeName: M20 - processArgs: {} - uriOptions: - retryReads: true -driverWorkload: - database: test_database - collection: test_collection - testData: - - {_id: 1, x: 11} - - {_id: 2, x: 22} - - {_id: 3, x: 33} - operations: - - - object: collection - name: find - arguments: - filter: - _id: {$gt: 1} - sort: - _id: 1 - result: - - {_id: 2, x: 22} - - {_id: 3, x: 33} diff --git a/tests/retryReads-resizeCluster.yml b/tests/retryReads-resizeCluster.yml new file mode 100644 index 00000000..60bbfb80 --- /dev/null +++ b/tests/retryReads-resizeCluster.yml @@ -0,0 +1,84 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + +operations: + - + setClusterConfiguration: + clusterConfiguration: + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M20 + processArgs: {} + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-testFailover-sharded.yml b/tests/retryReads-testFailover-sharded.yml new file mode 100644 index 00000000..bbeea866 --- /dev/null +++ b/tests/retryReads-testFailover-sharded.yml @@ -0,0 +1,82 @@ +initialConfiguration: + clusterConfiguration: + clusterType: SHARDED + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + +operations: + - + testFailover: true + - + sleep: 10 + - + waitForIdle: true + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-testFailover.yml b/tests/retryReads-testFailover.yml new file mode 100644 index 00000000..29f242aa --- /dev/null +++ b/tests/retryReads-testFailover.yml @@ -0,0 +1,82 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + +operations: + - + testFailover: true + - + sleep: 10 + - + waitForIdle: true + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-toggleServerSideJS.yaml b/tests/retryReads-toggleServerSideJS.yaml deleted file mode 100644 index cf641ee1..00000000 --- a/tests/retryReads-toggleServerSideJS.yaml +++ /dev/null @@ -1,35 +0,0 @@ -maintenancePlan: - initial: - clusterConfiguration: - clusterType: REPLICASET - providerSettings: - providerName: AWS - regionName: US_WEST_1 - instanceSizeName: M10 - processArgs: - javascriptEnabled: false - final: - clusterConfiguration: {} - processArgs: - javascriptEnabled: true - uriOptions: - retryReads: true -driverWorkload: - database: test_database - collection: test_collection - testData: - - {_id: 1, x: 11} - - {_id: 2, x: 22} - - {_id: 3, x: 33} - operations: - - - object: collection - name: find - arguments: - filter: - _id: {$gt: 1} - sort: - _id: 1 - result: - - {_id: 2, x: 22} - - {_id: 3, x: 33} diff --git a/tests/retryReads-toggleServerSideJS.yml b/tests/retryReads-toggleServerSideJS.yml new file mode 100644 index 00000000..42ece6a5 --- /dev/null +++ b/tests/retryReads-toggleServerSideJS.yml @@ -0,0 +1,81 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + +operations: + - + setClusterConfiguration: + clusterConfiguration: {} + processArgs: + javascriptEnabled: true + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-vmRestart-sharded.yml b/tests/retryReads-vmRestart-sharded.yml new file mode 100644 index 00000000..1a85f70b --- /dev/null +++ b/tests/retryReads-vmRestart-sharded.yml @@ -0,0 +1,82 @@ +initialConfiguration: + clusterConfiguration: + clusterType: SHARDED + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + +operations: + - + restartVms: true + - + sleep: 10 + - + waitForIdle: true + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryReads-vmRestart.yml b/tests/retryReads-vmRestart.yml new file mode 100644 index 00000000..bd031696 --- /dev/null +++ b/tests/retryReads-vmRestart.yml @@ -0,0 +1,82 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + +operations: + - + restartVms: true + - + sleep: 10 + - + waitForIdle: true + +driverWorkload: + description: "Find" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryReads: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + + tests: + - description: "Find one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 22 + - + _id: 3 + x: 33 diff --git a/tests/retryWrites-resizeCluster.yaml b/tests/retryWrites-resizeCluster.yaml deleted file mode 100644 index 91ff9152..00000000 --- a/tests/retryWrites-resizeCluster.yaml +++ /dev/null @@ -1,27 +0,0 @@ -maintenancePlan: - initial: - clusterConfiguration: - clusterType: REPLICASET - providerSettings: - providerName: AWS - regionName: US_WEST_1 - instanceSizeName: M10 - processArgs: {} - final: - clusterConfiguration: - providerSettings: - providerName: AWS - regionName: US_WEST_1 - instanceSizeName: M20 - processArgs: {} - uriOptions: - retryWrites: true -driverWorkload: - database: test_database - collection: test_collection - operations: - - - object: collection - name: insertOne - arguments: - document: {data: 100} diff --git a/tests/retryWrites-resizeCluster.yml b/tests/retryWrites-resizeCluster.yml new file mode 100644 index 00000000..c280957a --- /dev/null +++ b/tests/retryWrites-resizeCluster.yml @@ -0,0 +1,68 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + +operations: + - + setClusterConfiguration: + clusterConfiguration: + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M20 + processArgs: {} + +driverWorkload: + description: "Insert" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryWrites: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + tests: + - description: "Insert one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: insertOne + object: *collection0 + arguments: + document: { data: 100 } diff --git a/tests/retryWrites-toggleServerSideJS.yaml b/tests/retryWrites-toggleServerSideJS.yaml deleted file mode 100644 index fb17175c..00000000 --- a/tests/retryWrites-toggleServerSideJS.yaml +++ /dev/null @@ -1,25 +0,0 @@ -maintenancePlan: - initial: - clusterConfiguration: - clusterType: REPLICASET - providerSettings: - providerName: AWS - regionName: US_WEST_1 - instanceSizeName: M10 - processArgs: - javascriptEnabled: false - final: - clusterConfiguration: {} - processArgs: - javascriptEnabled: true - uriOptions: - retryWrites: true -driverWorkload: - database: test_database - collection: test_collection - operations: - - - object: collection - name: insertOne - arguments: - document: {data: 100} diff --git a/tests/retryWrites-toggleServerSideJS.yml b/tests/retryWrites-toggleServerSideJS.yml new file mode 100644 index 00000000..18133b5a --- /dev/null +++ b/tests/retryWrites-toggleServerSideJS.yml @@ -0,0 +1,65 @@ +initialConfiguration: + clusterConfiguration: + clusterType: REPLICASET + providerSettings: + providerName: AWS + regionName: US_WEST_1 + instanceSizeName: M10 + processArgs: {} + +operations: + - + setClusterConfiguration: + clusterConfiguration: {} + processArgs: + javascriptEnabled: true + +driverWorkload: + description: "Insert" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + uriOptions: + retryWrites: true + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + tests: + - description: "Insert one" + operations: + - name: loop + object: testRunner + arguments: + storeErrorsAsEntity: errors + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: insertOne + object: *collection0 + arguments: + document: { data: 100 } diff --git a/tests/validator-numErrors.yml b/tests/validator-numErrors.yml new file mode 100644 index 00000000..7959ea69 --- /dev/null +++ b/tests/validator-numErrors.yml @@ -0,0 +1,51 @@ +# This file intentionally causes the workload executor to produce an error +# on each execution. + +operations: [] + +driverWorkload: + description: "Validator - num errors" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - + _id: validation_sentinel + count: 0 + + tests: + - description: "updateOne & error" + operations: + - name: loop + object: testRunner + arguments: + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + storeErrorsAsEntity: errors + operations: + - name: updateOne + object: *collection0 + arguments: + filter: { _id: validation_sentinel} + update: + $inc: + count: 1 + - name: doesNotExist + object: *collection0 + arguments: + foo: bar diff --git a/tests/validator-numFailures-as-errors.yml b/tests/validator-numFailures-as-errors.yml new file mode 100644 index 00000000..f5c96ced --- /dev/null +++ b/tests/validator-numFailures-as-errors.yml @@ -0,0 +1,58 @@ +# This file intentionally causes the workload executor to produce a failure +# on each execution. + +operations: [] + +driverWorkload: + description: "Validator - num failures" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - + _id: 2 + x: 2 + + tests: + - description: "Find one & failure" + operations: + - name: loop + object: testRunner + arguments: + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + storeErrorsAsEntity: errors + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 2 + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 42 diff --git a/tests/validator-numFailures.yml b/tests/validator-numFailures.yml new file mode 100644 index 00000000..c8a4d391 --- /dev/null +++ b/tests/validator-numFailures.yml @@ -0,0 +1,58 @@ +# This file intentionally causes the workload executor to produce a failure +# on each execution. + +operations: [] + +driverWorkload: + description: "Validator - num failures" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - + _id: 2 + x: 2 + + tests: + - description: "Find one & failure" + operations: + - name: loop + object: testRunner + arguments: + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + storeFailuresAsEntity: failures + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 2 + - name: find + object: *collection0 + arguments: + filter: { _id: { $gt: 1 }} + sort: { _id: 1 } + expectResult: + - + _id: 2 + x: 42 diff --git a/tests/validator-simple.yml b/tests/validator-simple.yml new file mode 100644 index 00000000..1a75bab8 --- /dev/null +++ b/tests/validator-simple.yml @@ -0,0 +1,59 @@ +operations: [] + +driverWorkload: + description: "Validator - simple" + + schemaVersion: "1.0" + + createEntities: + - client: + id: &client0 client0 + storeEventsAsEntities: + events: + - PoolCreatedEvent + - PoolReadyEvent + - PoolClearedEvent + - PoolClosedEvent + - ConnectionCreatedEvent + - ConnectionReadyEvent + - ConnectionClosedEvent + - ConnectionCheckOutStartedEvent + - ConnectionCheckOutFailedEvent + - ConnectionCheckedOutEvent + - ConnectionCheckedInEvent + - CommandStartedEvent + - CommandSucceededEvent + - CommandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name dat + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name dat + + initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - + _id: validation_sentinel + count: 0 + + tests: + - description: "updateOne" + operations: + - name: loop + object: testRunner + arguments: + storeIterationsAsEntity: iterations + storeSuccessesAsEntity: successes + operations: + - name: updateOne + object: *collection0 + arguments: + filter: { _id: validation_sentinel} + update: + $inc: + count: 1