From f3ea1650dcc9c0b726a139aa4818d99cd386954f Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Thu, 5 Dec 2024 11:12:28 -0600 Subject: [PATCH] enable python integration tests. (#1024) * improve cli functionality, list account by id, export to named file, list address by index, parsable list balance for accounts. * use host network for devcontainer * improve startup scripts for sync jobs, charts for same namespace deployment * reduce amount sent in tests * create helpers and add python integration tests into workflow * use network_status calls and use "all" instead of manually calculating fees --- .devcontainer.json | 9 +- .../test-python-integration/action.yaml | 114 ++++++++ .github/workflows/on-pr.yaml | 29 ++ .internal-ci/docker/Dockerfile.full-service | 3 +- .../docker/entrypoints/full-service.sh | 9 + .../templates/full-service-service.yaml | 2 +- .../templates/full-service-statefulSet.yaml | 4 +- .internal-ci/util/wait-for-full-service.sh | 36 ++- python/mobilecoin/cli.py | 105 +++++--- python/tests/conftest.py | 2 +- python/tests/test_client.py | 22 +- python/tests/test_client_v1.py | 25 +- tools/.shared-functions.sh | 73 ++++- tools/run-fs.sh | 31 ++- tools/test-python-integration.sh | 255 ++++++++++++++++++ 15 files changed, 629 insertions(+), 90 deletions(-) create mode 100644 .github/actions/test-python-integration/action.yaml create mode 100755 tools/test-python-integration.sh diff --git a/.devcontainer.json b/.devcontainer.json index b933bd0dc..16c2aa49d 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -1,13 +1,13 @@ { "image": "mobilecoin/fat-devcontainer:v0.0.38", - "forwardPorts": [ - 9090 + "runArgs": [ + "--network=host" ], "capAdd": ["SYS_PTRACE"], "containerEnv": { "MC_CHAIN_ID": "local", "RUST_BACKTRACE": "1", - "SGX_MODE": "SW" + "SGX_MODE": "HW" }, "remoteUser": "sentz", "postCreateCommand": "/usr/local/bin/startup-devcontainer.sh", @@ -21,7 +21,8 @@ "rust-lang.rust-analyzer", "timonwong.shellcheck", "be5invis.toml", - "redhat.vscode-yaml" + "redhat.vscode-yaml", + "ms-python.python" ] } } diff --git a/.github/actions/test-python-integration/action.yaml b/.github/actions/test-python-integration/action.yaml new file mode 100644 index 000000000..c9c974d53 --- /dev/null +++ b/.github/actions/test-python-integration/action.yaml @@ -0,0 +1,114 @@ +name: Test - Python Integration +description: Set up environment and run integration tests + +inputs: + network: + description: "main|test" + required: true + cache_buster: + description: "cache buster" + required: true + version: + description: "Version of the full-service to test" + required: true + rancher_cluster: + description: "Rancher cluster to deploy to" + required: true + rancher_url: + description: "Rancher URL" + required: true + rancher_token: + description: "Rancher token" + required: true + +runs: + using: composite + steps: + - name: Install pip + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y python3-pip + + # Deploy fs chart with cloned volume + # All tests will need to be deployed in the full-service-ledger namespace so we can clone the target PVC + - name: Generate full-service values file + shell: bash + run: | + mkdir -p .mob/ + cat < .mob/${{ inputs.network }}.values.yaml + fullService: + persistence: + enabled: true + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 128Gi + dataSource: + name: full-service-ledger-${{ inputs.network }}net + kind: PersistentVolumeClaim + EOF + + - name: Deploy Full-Service + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: helm-deploy + chart_repo: https://harbor.mobilecoin.com/chartrepo/mobilecoinofficial-public + chart_name: full-service + chart_version: ${{ inputs.version }}-${{ inputs.network }}net + chart_values: .mob/${{ inputs.network }}.values.yaml + chart_wait_timeout: 30m + release_name: ${{ inputs.version }}-${{ inputs.network }}net + namespace: full-service-ledger + rancher_cluster: ${{ inputs.rancher_cluster }} + rancher_url: ${{ inputs.rancher_url }} + rancher_token: ${{ inputs.rancher_token }} + + - name: Get IP address of full-service + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: kubectl-exec + rancher_cluster: ${{ inputs.rancher_cluster }} + rancher_url: ${{ inputs.rancher_url }} + rancher_token: ${{ inputs.rancher_token }} + command: | + target_ip=$(kubectl -n full-service-ledger get svc ${{ inputs.version }}-${{ inputs.network }}net-full-service -o jsonpath='{.spec.clusterIP}') + funding_ip=$(kubectl -n dev-wallet-${{ inputs.network }}net get svc full-service -o jsonpath='{.spec.clusterIP}') + echo "TARGET_IP=${target_ip}" >> "${GITHUB_ENV}" + echo "FUNDING_IP=${funding_ip}" >> "${GITHUB_ENV}" + + - name: Run Integration Tests + env: + FUNDING_FS_URL: http://${{ env.FUNDING_IP }}:9090/wallet/v2 + TARGET_FS_URL: http://${{ env.TARGET_IP }}:9090/wallet/v2 + shell: bash + run: | + set -e + + # Wait for the full-service to be ready + .internal-ci/util/wait-for-full-service.sh + + # Run integration tests + ./tools/test-python-integration.sh ${{ inputs.network }} + + - name: Cleanup helm chart + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: helm-release-delete + release_name: ${{ inputs.version }}-${{ inputs.network }}net + namespace: full-service-ledger + rancher_cluster: ${{ inputs.rancher_cluster }} + rancher_url: ${{ inputs.rancher_url }} + rancher_token: ${{ inputs.rancher_token }} + + - name: Cleanup PVC + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: pvc-delete + namespace: full-service-ledger + object_name: data-${{ inputs.version }}-${{ inputs.network }}net-full-service-0 + rancher_cluster: ${{ inputs.rancher_cluster }} + rancher_url: ${{ inputs.rancher_url }} + rancher_token: ${{ inputs.rancher_token }} diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index c7906e6a1..c1129ece2 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -223,6 +223,7 @@ jobs: - meta - build-rust-linux strategy: + fail-fast: false matrix: runner: - mco-dev-small-x64 @@ -248,6 +249,7 @@ jobs: - meta - build-containers strategy: + fail-fast: false matrix: network: - main @@ -270,6 +272,7 @@ jobs: - meta - publish-containers strategy: + fail-fast: false matrix: network: - main @@ -286,6 +289,31 @@ jobs: repo_username: ${{ secrets.HARBOR_USERNAME }} repo_password: ${{ secrets.HARBOR_PASSWORD }} + test-python-integration: + needs: + - meta + - build-publish-charts + strategy: + fail-fast: false + matrix: + network: + - main + - test + runs-on: mco-dev-small-x64 + steps: + - name: Checkout + uses: mobilecoinofficial/gh-actions/checkout@v0 + + - name: Run Python Integration Tests + uses: ./.github/actions/test-python-integration + with: + version: ${{ needs.meta.outputs.version }} + network: ${{ matrix.network }} + cache_buster: ${{ vars.CACHE_BUSTER }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + checks-successful: needs: - lint-actions @@ -296,6 +324,7 @@ jobs: - test-rust - build-rust-macos - build-publish-charts + - test-python-integration runs-on: mco-dev-small-x64 steps: - name: Success diff --git a/.internal-ci/docker/Dockerfile.full-service b/.internal-ci/docker/Dockerfile.full-service index b10d56e9e..cf90933ef 100644 --- a/.internal-ci/docker/Dockerfile.full-service +++ b/.internal-ci/docker/Dockerfile.full-service @@ -17,7 +17,7 @@ RUN addgroup --system --gid 1000 app \ RUN apt-get update \ && apt-get upgrade -y \ - && apt-get install -y ca-certificates curl libdbus-1-3 libusb-1.0-0 \ + && apt-get install -y jq ca-certificates curl libdbus-1-3 libusb-1.0-0 \ && apt-get clean \ && rm -r /var/lib/apt/lists/* \ && mkdir -p /usr/share/grpc \ @@ -33,6 +33,7 @@ COPY ${RUST_BIN_PATH}/wallet-service-mirror-public /usr/local/bin/ COPY ${RUST_BIN_PATH}/generate-rsa-keypair /usr/local/bin/ COPY ${RUST_BIN_PATH}/ingest-enclave.css /usr/local/bin/ COPY .internal-ci/docker/entrypoints/full-service.sh /usr/local/bin/entrypoint.sh +COPY .internal-ci/util/wait-for-full-service.sh /usr/local/bin/wait-for-full-service.sh # not implemented yet # COPY .internal-ci/docker/support/full-service/initialize-wallets.sh /usr/local/bin/initialize-wallets.sh diff --git a/.internal-ci/docker/entrypoints/full-service.sh b/.internal-ci/docker/entrypoints/full-service.sh index 1baa0a957..999a24f10 100755 --- a/.internal-ci/docker/entrypoints/full-service.sh +++ b/.internal-ci/docker/entrypoints/full-service.sh @@ -41,6 +41,15 @@ then fi fi +# Run until we have a valid ledger database then quit. +if [[ -n "${SYNC_LEDGER_ONLY}" ]] +then + echo "SYNC_LEDGER_ONLY set. Exiting after syncing ledger." + RUST_LOG=error /usr/local/bin/full-service & + /usr/local/bin/wait-for-full-service.sh + exit 0 +fi + # Check to see if leading argument starts with "--". # If so execute with full-service for compatibility with the previous container/cli arg only configuration. if [[ "${1}" =~ ^--.* ]] diff --git a/.internal-ci/helm/full-service/templates/full-service-service.yaml b/.internal-ci/helm/full-service/templates/full-service-service.yaml index a0d479f86..323f6746f 100644 --- a/.internal-ci/helm/full-service/templates/full-service-service.yaml +++ b/.internal-ci/helm/full-service/templates/full-service-service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: full-service + name: {{ include "fullService.fullname" . }} labels: {{- include "fullService.labels" . | nindent 4 }} spec: diff --git a/.internal-ci/helm/full-service/templates/full-service-statefulSet.yaml b/.internal-ci/helm/full-service/templates/full-service-statefulSet.yaml index f252205cc..03f6ffcac 100644 --- a/.internal-ci/helm/full-service/templates/full-service-statefulSet.yaml +++ b/.internal-ci/helm/full-service/templates/full-service-statefulSet.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ include "fullService.fullname" . }}-full-service + name: {{ include "fullService.fullname" . }} labels: {{- include "fullService.labels" . | nindent 4 }} spec: @@ -11,7 +11,7 @@ spec: matchLabels: app: full-service {{- include "fullService.selectorLabels" . | nindent 6 }} - serviceName: {{ include "fullService.fullname" . }}-full-service + serviceName: {{ include "fullService.fullname" . }} template: metadata: annotations: diff --git a/.internal-ci/util/wait-for-full-service.sh b/.internal-ci/util/wait-for-full-service.sh index aa304ba76..976cb5bc7 100755 --- a/.internal-ci/util/wait-for-full-service.sh +++ b/.internal-ci/util/wait-for-full-service.sh @@ -4,28 +4,42 @@ set -e set -o pipefail -echo "Checking block height - wait for full-service to start" -sleep 15 +echo "-- Checking block height - wait for full-service to start" + +TARGET_FS_URL=${TARGET_FS_URL:-"http://localhost:9090/wallet/v2"} curl_post() { - curl --connect-timeout 2 -sSL -X POST -H 'Content-type: application/json' http://localhost:9090/wallet/v2 --data '{"method": "get_wallet_status", "jsonrpc": "2.0", "id": 1}' 2>/dev/null + curl --connect-timeout 2 -sSfL -X POST -H 'Content-type: application/json' "${TARGET_FS_URL}" --data '{"method": "get_network_status", "jsonrpc": "2.0", "id": 1}' 2>/dev/null } +count=0 +while ! curl_post +do + echo "-- Waiting for full-service to respond" + count=$((count + 1)) + if [[ "${count}" -gt 300 ]] + then + echo "Full-service did not start" + exit 1 + fi + sleep 2 +done + # wait for blocks -wallet_json=$(curl_post) -network_block_height=$(echo "${wallet_json}" | jq -r .result.wallet_status.network_block_height) -local_block_height=$(echo "${wallet_json}" | jq -r .result.wallet_status.local_block_height) +network_json=$(curl_post) +network_block_height=$(echo "${network_json}" | jq -r .result.network_status.network_block_height) +local_block_height=$(echo "${network_json}" | jq -r .result.network_status.local_block_height) while [[ "${local_block_height}" != "${network_block_height}" ]] do - echo "- Waiting for blocks to download ${local_block_height} of ${network_block_height}" + echo "-- Waiting for blocks to download ${local_block_height} of ${network_block_height}" - wallet_json=$(curl_post) - network_block_height=$(echo "${wallet_json}" | jq -r .result.wallet_status.network_block_height) - local_block_height=$(echo "${wallet_json}" | jq -r .result.wallet_status.local_block_height) + network_json=$(curl_post) + network_block_height=$(echo "${network_json}" | jq -r .result.network_status.network_block_height) + local_block_height=$(echo "${network_json}" | jq -r .result.network_status.local_block_height) sleep 10 done -echo "full-service sync is done" +echo "-- full-service sync is done" diff --git a/python/mobilecoin/cli.py b/python/mobilecoin/cli.py index 7eae98d11..9e68c74e4 100644 --- a/python/mobilecoin/cli.py +++ b/python/mobilecoin/cli.py @@ -60,6 +60,7 @@ def _create_parsers(self): # List accounts. self.list_args = command_sp.add_parser('list', help='List accounts.') + self.list_args.add_argument('account_id', nargs='?', type=str, default=None, help='ID or name of the account to list.') # Create account. self.create_args = command_sp.add_parser('create', help='Create a new account.') @@ -95,6 +96,7 @@ def _create_parsers(self): self.export_args.add_argument('account_id', help='ID of the account to export.') self.export_args.add_argument('-s', '--show', action='store_true', help='Only show the secret entropy mnemonic, do not write it to file.') + self.export_args.add_argument('-f', '--file', type=str, default=None, help='Write the secret entropy mnemonic to a file path.') # Remove account. self.remove_args = command_sp.add_parser('remove', help='Remove an account from local storage.') @@ -110,7 +112,7 @@ def _create_parsers(self): self.send_args.add_argument('--build-only', action='store_true', help='Just build the transaction, do not submit it.') self.send_args.add_argument('--fee', type=str, default=None, help='The fee paid to the network.') self.send_args.add_argument('account_id', help='Source account ID.') - self.send_args.add_argument('amount', help='Amount to send.') + self.send_args.add_argument('amount', help='Amount to send. Use "all" to automatically calculate fees and send all available funds.') self.send_args.add_argument('token', help='Token to send (MOB, eUSD).') self.send_args.add_argument('to_address', help='Address to send to.') @@ -131,6 +133,7 @@ def _create_parsers(self): # List addresses. self.address_list_args = address_action.add_parser('list', help='List addresses and balances for an account.') self.address_list_args.add_argument('account_id', help='Account ID.') + self.address_list_args.add_argument('-i', '--index', nargs='?', type=int, default=None, help='Show only main address.') # Create address. self.address_create_args = address_action.add_parser( @@ -178,6 +181,11 @@ def _create_parsers(self): self.gift_remove_args = gift_action.add_parser('remove', help='Remove a gift code.') self.gift_remove_args.add_argument('gift_code', help='Gift code to remove.') + # List balance for account + self.balance_args = command_sp.add_parser('balance', help='get balance for account') + self.balance_args.add_argument('account_id', type=str, nargs='?', default=None, help='ID or name of the account to list, or empty for all accounts.') + self.balance_args.add_argument('-t', '--token', type=str, default='MOB', help='Token to get balance for (MOB, eUSD).') + # Version self.version_args = command_sp.add_parser('version', help='Show version number.') @@ -254,20 +262,44 @@ def status(self): ) print(indent(amount.format(), ' ')) - def list(self): - accounts = self.client.get_accounts() - if len(accounts) == 0: - print('No accounts.') - return + def list(self, account_id): + if account_id: + # Show a single account. + account = self._load_account_prefix(account_id) + status = self.client.get_account_status(account['id']) + _print_account(status) + else: + accounts = self.client.get_accounts() + if len(accounts) == 0: + print('No accounts.') + return - account_list = [] - for account_id in accounts.keys(): - status = self.client.get_account_status(account_id) - account_list.append(status) + account_list = [] + for account_id in accounts.keys(): + status = self.client.get_account_status(account_id) + account_list.append(status) - for status in account_list: - print() - _print_account(status) + for status in account_list: + print() + _print_account(status) + + def balance(self, account_id, token): + if account_id is None: + accounts = self.client.get_accounts() + for account_id in accounts.keys(): + self.balance(account_id, token) + return + account = self._load_account_prefix(account_id) + account_id = account['id'] + token = get_token(token) + + status = self.client.get_account_status(account_id) + balance = status['balance_per_token'].get(str(token.token_id)) + if balance is None: + print('{} 0 {} {}'.format(account_id[:6], token.short_code, _format_sync_state(status))) + return + balance = Amount.from_storage_units(balance['unspent'], token) + print('{} {} {}'.format(account_id[:6], balance.format(), _format_sync_state(status))) def create(self, name=None, disable_fog=False): if disable_fog: @@ -364,11 +396,16 @@ def import_hardware(self, name=None, disable_fog=False): print('Imported account.') print(_format_account_header(account)) - def export(self, account_id, show=False): + def export(self, account_id, file, show=False): account = self._load_account_prefix(account_id) account_id = account['id'] status = self.client.get_account_status(account_id) + if file is None: + filename = 'mobilecoin_secret_mnemonic_{}.json'.format(account_id[:6]) + else: + filename = file + print('You are about to export the secret entropy mnemonic for this account:') print() _print_account(status) @@ -383,7 +420,7 @@ def export(self, account_id, show=False): if show: confirm_message = 'Really show account entropy mnemonic? (Y/N) ' else: - confirm_message = 'Really write account entropy mnemonic to a file? (Y/N) ' + confirm_message = 'Really write account entropy mnemonic to file: {} (Y/N) '.format(filename) if not self.confirm(confirm_message): print('Cancelled.') return @@ -396,14 +433,13 @@ def export(self, account_id, show=False): print('{:<2} {}'.format(i, word)) print() else: - filename = 'mobilecoin_secret_mnemonic_{}.json'.format(account_id[:6]) try: _save_export(account, secrets, filename) except OSError as e: print('Could not write file: {}'.format(e)) exit(1) else: - print(f'Wrote {filename}.') + print(f'Wrote {filename}') def remove(self, account_id): account = self._load_account_prefix(account_id) @@ -484,7 +520,7 @@ def tx_block(t): txo['amount']['token_id'], ) print(indent(amount.format(), ' ')) - address = txo['recipient_public_address_b58'] + address = txo['recipient_public_address_b58'] if address in own_addresses: print(' Received at', address) else: @@ -529,7 +565,7 @@ def send(self, account_id, amount, token, to_address, build_only=False, fee=None token.short_code, account_id[:6], )) - return + exit(1) if build_only: verb = 'Building transaction for' @@ -700,27 +736,32 @@ def address(self, action, **args): else: func(**args) - def address_list(self, account_id): + def address_list(self, account_id, index): account = self._load_account_prefix(account_id) - print() - print(_format_account_header(account)) addresses = self.client.get_addresses(account['id'], limit=1000) addresses = list(addresses.values()) addresses.sort(key=lambda a: int(a['subaddress_index'])) - for address in addresses: - address_status = self.client.get_address_status(address['public_address_b58']) - + # only print headers if we're listing multiple accounts + if index is None: print() - print('#{} {}'.format( - address['subaddress_index'], - address['metadata'], - )) - print(indent(address['public_address_b58'], ' ')) - print(indent(_format_balances(address_status['balance_per_token']), ' ')) + print(_format_account_header(account)) - print() + for address in addresses: + address_status = self.client.get_address_status(address['public_address_b58']) + + print() + print('#{} {}'.format( + address['subaddress_index'], + address['metadata'], + )) + print(indent(address['public_address_b58'], ' ')) + print(indent(_format_balances(address_status['balance_per_token']), ' ')) + + print() + else: + print(addresses[index]['public_address_b58']) def address_create(self, account_id, metadata): account = self._load_account_prefix(account_id) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 12da4e13b..748ff4f12 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -85,7 +85,7 @@ async def source_account(client): status['balance_per_token'][str(MOB.token_id)]['unspent'], MOB, ) - assert initial_balance >= Amount.from_display_units(0.1, MOB) + assert initial_balance >= Amount.from_display_units(0.001, MOB) return status['account'] diff --git a/python/tests/test_client.py b/python/tests/test_client.py index a6f656713..6ca279caa 100644 --- a/python/tests/test_client.py +++ b/python/tests/test_client.py @@ -106,14 +106,14 @@ async def test_send_transaction_self(client, source_account, fees): # Send a transaction from the account back to itself. transaction_log, _ = await client.build_and_submit_transaction( source_account['id'], - Amount.from_display_units(0.01, MOB), + Amount.from_display_units(0.001, MOB), source_account['main_address'], ) tx_value = Amount.from_storage_units( transaction_log['value_map'][str(MOB.token_id)], MOB, ) - assert tx_value == Amount.from_display_units(0.01, MOB) + assert tx_value == Amount.from_display_units(0.001, MOB) # Wait for the account to sync. tx_index = int(transaction_log['submitted_block_index']) @@ -131,14 +131,14 @@ async def _test_send_transaction(client, account, temp_account): # Send a transaction to the temporary account. transaction_log, _ = await client.build_and_submit_transaction( account['id'], - Amount.from_display_units(0.01, MOB), + Amount.from_display_units(0.001, MOB), temp_account['main_address'], ) tx_value = Amount.from_storage_units( transaction_log['value_map'][str(MOB.token_id)], MOB, ) - assert tx_value == Amount.from_display_units(0.01, MOB) + assert tx_value == Amount.from_display_units(0.001, MOB) assert transaction_log['output_txos'][0]['public_key'] == transaction_log['id'] # Wait for the temporary account to sync. @@ -150,7 +150,7 @@ async def _test_send_transaction(client, account, temp_account): temp_status['balance_per_token'][str(MOB.token_id)]['unspent'], MOB, ) - assert temp_balance == Amount.from_display_units(0.01, MOB) + assert temp_balance == Amount.from_display_units(0.001, MOB) async def test_send_transaction(client, source_account, account_factory): @@ -180,7 +180,7 @@ async def test_send_transaction_subaddress(client, source_account, account_facto # Send a transaction to the temporary account. transaction_log, _ = await client.build_and_submit_transaction( source_account['id'], - Amount.from_display_units(0.01, MOB), + Amount.from_display_units(0.001, MOB), address, ) @@ -194,7 +194,7 @@ async def test_send_transaction_subaddress(client, source_account, account_facto temp_status['balance_per_token'][str(MOB.token_id)]['unspent'], MOB, ) - assert temp_balance == Amount.from_display_units(0.01, MOB) + assert temp_balance == Amount.from_display_units(0.001, MOB) # The subaddress balance also shows the transaction. subaddress_status = await client.get_address_status(address) @@ -202,7 +202,7 @@ async def test_send_transaction_subaddress(client, source_account, account_facto subaddress_status['balance_per_token'][str(MOB.token_id)]['unspent'], MOB, ) - assert subaddress_balance == Amount.from_display_units(0.01, MOB) + assert subaddress_balance == Amount.from_display_units(0.001, MOB) async def test_build_transaction_multiple_outputs(client, source_account, account_factory): @@ -212,8 +212,8 @@ async def test_build_transaction_multiple_outputs(client, source_account, accoun tx_proposal, _ = await client.build_transaction( source_account['id'], { - temp_account_1['main_address']: Amount.from_display_units(0.01, MOB), - temp_account_2['main_address']: Amount.from_display_units(0.01, MOB), + temp_account_1['main_address']: Amount.from_display_units(0.001, MOB), + temp_account_2['main_address']: Amount.from_display_units(0.001, MOB), }, ) transaction_log = await client.submit_transaction(tx_proposal, source_account['id']) @@ -231,7 +231,7 @@ async def test_build_transaction_multiple_outputs(client, source_account, accoun status['balance_per_token'][str(MOB.token_id)]['unspent'], MOB, ) - assert balance == Amount.from_display_units(0.01, MOB) + assert balance == Amount.from_display_units(0.001, MOB) async def test_build_burn_transaction(client, source_account, fees): diff --git a/python/tests/test_client_v1.py b/python/tests/test_client_v1.py index af8f81a18..41ae29796 100644 --- a/python/tests/test_client_v1.py +++ b/python/tests/test_client_v1.py @@ -109,11 +109,11 @@ def test_send_transaction_self(client_v1, source_account, fee): # Send a transaction from the account back to itself. transaction_log, _ = client_v1.build_and_submit_transaction( source_account['id'], - Amount.from_display_units(0.01, MOB), + Amount.from_display_units(0.001, MOB), source_account['main_address'], ) tx_value = Amount.from_storage_units(transaction_log['value_pmob'], MOB) - assert tx_value == Amount.from_display_units(0.01, MOB) + assert tx_value == Amount.from_display_units(0.001, MOB) # Wait for the account to sync. tx_index = int(transaction_log['submitted_block_index']) @@ -128,11 +128,11 @@ def _test_send_transaction(client_v1, account, temp_account): # Send a transaction to the temporary account. transaction_log, _ = client_v1.build_and_submit_transaction( account['id'], - Amount.from_display_units(0.01, MOB), + Amount.from_display_units(0.001, MOB), temp_account['main_address'], ) tx_value = Amount.from_storage_units(transaction_log['value_pmob'], MOB) - assert tx_value == Amount.from_display_units(0.01, MOB) + assert tx_value == Amount.from_display_units(0.001, MOB) # Wait for the temporary account to sync. tx_index = int(transaction_log['submitted_block_index']) @@ -143,7 +143,7 @@ def _test_send_transaction(client_v1, account, temp_account): temp_balance['unspent_pmob'], MOB, ) - assert temp_balance == Amount.from_display_units(0.01, MOB) + assert temp_balance == Amount.from_display_units(0.001, MOB) async def test_send_transaction(client_v1, source_account, account_factory): @@ -162,7 +162,7 @@ async def test_send_transaction_fog(client_v1, source_account, account_factory): temp_fog_account = await account_factory.create_fog() _test_send_transaction(client_v1, source_account, temp_fog_account) - +# failed - not waiting for primary account to be synced async def test_send_transaction_subaddress(client_v1, source_account, account_factory): temp_account = await account_factory.create() @@ -178,7 +178,7 @@ async def test_send_transaction_subaddress(client_v1, source_account, account_fa # Send a transaction to the temporary account. transaction_log, _ = client_v1.build_and_submit_transaction( source_account['id'], - Amount.from_display_units(0.01, MOB), + Amount.from_display_units(0.001, MOB), address, ) @@ -188,13 +188,12 @@ async def test_send_transaction_subaddress(client_v1, source_account, account_fa # Check that the transaction has arrived. temp_balance = Amount.from_storage_units(temp_balance['unspent_pmob'], MOB) - assert temp_balance == Amount.from_display_units(0.01, MOB) + assert temp_balance == Amount.from_display_units(0.001, MOB) # The subaddress balance also shows the transaction. balance = client_v1.get_balance_for_address(address) subaddress_balance = Amount.from_storage_units(balance['unspent_pmob'], MOB) - assert subaddress_balance == Amount.from_display_units(0.01, MOB) - + assert subaddress_balance == Amount.from_display_units(0.001, MOB) async def test_build_transaction_multiple_outputs(client_v1, source_account, account_factory): temp_account_1 = await account_factory.create() @@ -203,8 +202,8 @@ async def test_build_transaction_multiple_outputs(client_v1, source_account, acc tx_proposal, _ = client_v1.build_transaction( source_account['id'], { - temp_account_1['main_address']: Amount.from_display_units(0.01, MOB), - temp_account_2['main_address']: Amount.from_display_units(0.01, MOB), + temp_account_1['main_address']: Amount.from_display_units(0.001, MOB), + temp_account_2['main_address']: Amount.from_display_units(0.001, MOB), }, ) transaction_log = client_v1.submit_transaction( @@ -225,4 +224,4 @@ async def test_build_transaction_multiple_outputs(client_v1, source_account, acc balance['unspent_pmob'], MOB, ) - assert balance == Amount.from_display_units(0.01, MOB) + assert balance == Amount.from_display_units(0.001, MOB) diff --git a/tools/.shared-functions.sh b/tools/.shared-functions.sh index 28427840f..68b8e2ce5 100755 --- a/tools/.shared-functions.sh +++ b/tools/.shared-functions.sh @@ -6,11 +6,17 @@ GIT_BASE=$(git rev-parse --show-toplevel) AM_I_IN_MOB_PROMPT="no" # Assume that if you're git directory is /tmp/mobilenode that we're in mob prompt -if [[ "${GIT_BASE}" == "/tmp/mobilenode" ]] +if [[ "${GIT_BASE}" == "/tmp/mobilenode" || "${GIT_BASE}" == "/workspaces/full-service" ]] then AM_I_IN_MOB_PROMPT="yes" fi +if [[ -z "${net}" ]] +then + echo "ERROR: is not set" + exit 1 +fi + if [[ "${AM_I_IN_MOB_PROMPT}" == "yes" ]] then # Set cargo target dir to include the "net" @@ -20,8 +26,8 @@ then WORK_DIR=${WORK_DIR:-"${GIT_BASE}/.mob/${net}"} LISTEN_ADDR="0.0.0.0" else - CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-"${GIT_BASE}/target"} - WORK_DIR=${WORK_DIR:-"${HOME}/.mobilecoin/${net}"} + CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-"${GIT_BASE}/target/${net}"} + WORK_DIR=${WORK_DIR:-"${GIT_BASE}/.mob/${net}"} LISTEN_ADDR="127.0.0.1" fi @@ -31,6 +37,12 @@ export CARGO_TARGET_DIR RELEASE_DIR WORK_DIR LISTEN_ADDR # Setup release dir - set in .shared-functions.sh mkdir -p "${RELEASE_DIR}" +# Setup wallet dbs. Don't put them in target so they don't get cleaned up by cargo clean. +WALLET_DB_DIR="${WALLET_DB_DIR:-".mob/${net}-db/wallet-db"}" +LEDGER_DB_DIR="${LEDGER_DB_DIR:-".mob/${net}-db/ledger-db"}" +mkdir -p "${WALLET_DB_DIR}" +mkdir -p "${LEDGER_DB_DIR}" + if [[ "${AM_I_IN_MOB_PROMPT}" == "yes" ]] then # migrate wallet/ledger db to release_dir and remove workdir to make room @@ -39,11 +51,11 @@ then then if [[ -d "${WORK_DIR}/wallet-db" ]] then - mv "${WORK_DIR}/wallet-db" "${RELEASE_DIR}/wallet-db" + mv "${WORK_DIR}/wallet-db" "${WALLET_DB_DIR}" fi if [[ -d "${WORK_DIR}/ledger-db" ]] then - mv "${WORK_DIR}/ledger-db" "${RELEASE_DIR}/ledger-db" + mv "${WORK_DIR}/ledger-db" "${LEDGER_DB_DIR}" fi rm -rf "${WORK_DIR}" fi @@ -114,3 +126,54 @@ check_pid_file() fi fi } + +function parse_url() +{ + # 1: url + # 2: variable name to store the result + local varname=$2 + debug "varname: $varname" + + local proto full_url user hostport host port path + # extract the protocol + proto=$(echo "$1" | grep :// | sed -e's,^\(.*://\).*,\1,g') + debug "proto: $proto" + # remove the protocol + full_url="${1/$proto/}" + debug "url: $full_url" + # extract the user (if any) + if [[ "${full_url}" =~ '@' ]] + then + user=$(echo "${full_url}" | cut -d@ -f1) + else + user="" + fi + debug "user: $user" + # extract the host and port + hostport=$(echo "${full_url/$user@/}" | cut -d/ -f1) + debug "host and port: $hostport" + # by request host without port + host="${hostport/:*/}" + debug "host: $host" + # by request - try to extract the port + port=$(echo "${hostport}" | sed -e 's,^.*:,:,g' -e 's,.*:\([0-9]*\).*,\1,g' -e 's,[^0-9],,g') + debug "port: $port" + # extract the path (if any) + path=$(echo "${full_url}" | grep / | cut -d/ -f2-) + debug "path: $path" + + # shellcheck disable=SC1087 # this is a dynamic variable name doen't seem to work with brackets. + declare -g "$varname[proto]=${proto}" + # shellcheck disable=SC1087 + declare -g "$varname[url]=${full_url}" + # shellcheck disable=SC1087 + declare -g "$varname[user]=${user}" + # shellcheck disable=SC1087 + declare -g "$varname[hostport]=${hostport}" + # shellcheck disable=SC1087 + declare -g "$varname[port]=${port}" + # shellcheck disable=SC1087 + declare -g "$varname[host]=${host}" + # shellcheck disable=SC1087 + declare -g "$varname[path]=${path}" +} diff --git a/tools/run-fs.sh b/tools/run-fs.sh index f2ea579a4..56002c70b 100755 --- a/tools/run-fs.sh +++ b/tools/run-fs.sh @@ -36,6 +36,10 @@ do build=1 shift 1 ;; + --download-ledger) + download_ledger=1 + shift 1 + ;; --offline) export MC_OFFLINE=true shift 1 @@ -70,21 +74,16 @@ location=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) # shellcheck source=/dev/null source "${location}/.shared-functions.sh" -# Set default database directories -WALLET_DB_DIR="${WALLET_DB_DIR:-"${RELEASE_DIR}/wallet-db"}" -LEDGER_DB_DIR="${LEDGER_DB_DIR:-"${RELEASE_DIR}/ledger-db"}" -mkdir -p "${WALLET_DB_DIR}" -mkdir -p "${LEDGER_DB_DIR}" - # Set vars for all networks -MC_WALLET_DB="${WALLET_DB_DIR}/wallet.db" -MC_LEDGER_DB="${LEDGER_DB_DIR}" +MC_WALLET_DB="${WALLET_DB_DIR:?}/wallet.db" +MC_LEDGER_DB="${LEDGER_DB_DIR:?}" MC_LISTEN_HOST="${LISTEN_ADDR:?}" case "${net}" in test) domain_name="test.mobilecoin.com" tx_source_base="https://s3-us-west-1.amazonaws.com/mobilecoin.chain" + ledger_source="https://mcdevus1ledger.blob.core.windows.net/test/data.mdb" # Set chain id, peer and tx_sources for 2 nodes. MC_CHAIN_ID="${net}" @@ -98,6 +97,7 @@ case "${net}" in main) domain_name="prod.mobilecoinww.com" tx_source_base="https://ledger.mobilecoinww.com" + ledger_source="https://mcdeveu1ledger.blob.core.windows.net/main/data.mdb" # Set chain id, peer and tx_sources for 2 nodes. MC_CHAIN_ID="${net}" @@ -163,6 +163,19 @@ then "${location}/build-fs.sh" "${net}" fi +if [[ -n "${ledger_source}" && -n "${download_ledger}" ]] +then + if [[ -e "${MC_LEDGER_DB}/data.mdb" ]] + then + echo "Ledger already exists at ${MC_LEDGER_DB}/data.mdb" + echo "Remove ${MC_LEDGER_DB}/data.mdb or remove --download-ledger flag" + exit 1 + else + echo "Downloading ledger from ${ledger_source}" + curl -SLf "${ledger_source}" -o "${MC_LEDGER_DB}/data.mdb" + fi +fi + # start validator and unset envs for full-service if [[ -n "${validator}" ]] then @@ -179,7 +192,7 @@ then fi echo "Starting validator-service. Log is at /tmp/validator-service.log" - validator_ledger_db="${RELEASE_DIR}/validator/ledger-db" + validator_ledger_db="${GIT_BASE}/.mob/${net}-db/validator/ledger-db" mkdir -p "${validator_ledger_db}" # Override diff --git a/tools/test-python-integration.sh b/tools/test-python-integration.sh new file mode 100755 index 000000000..659146982 --- /dev/null +++ b/tools/test-python-integration.sh @@ -0,0 +1,255 @@ +#!/bin/bash + +set -e -o pipefail + +usage() +{ + echo "Usage:" + echo "${0} |check" + echo " - main|prod|test" + echo "Environment Variables:" + echo " FUNDING_FS_URL - optional: URL of the funding full-service" + echo " FUNDING_ACCOUNT_ID - optional: account_id of the funding account" + echo " TARGET_FS_URL - optional: URL of the full-service we want to test" +} + +while (( "$#" )) +do + case "${1}" in + --help | -h) + usage + exit 0 + ;; + *) + net="${1}" + shift 1 + ;; + esac +done + +if [[ -z "${net}" ]] +then + echo "ERROR: is not set" + usage + exit 1 +fi + +# use main instead of legacy prod +if [[ "${net}" == "prod" ]] +then + echo "Detected \"prod\" legacy network setting. Using \"main\" instead." + net=main +fi + +# Grab current location and source the shared functions. +location=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +# shellcheck source=/dev/null +source "${location}/.shared-functions.sh" + +case "${net}" in + test) + export MC_FOG_REPORT_URL="fog://fog.test.mobilecoin.com" + export MC_WALLET_FILE=${MC_WALLET_FILE:-${GIT_BASE}/.mob/testnet_secret_mnemonic.json} + export MC_FOG_AUTHORITY_SPKI="MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvnB9wTbTOT5uoizRYaYbw7XIEkInl8E7MGOAQj+xnC+F1rIXiCnc/t1+5IIWjbRGhWzo7RAwI5sRajn2sT4rRn9NXbOzZMvIqE4hmhmEzy1YQNDnfALAWNQ+WBbYGW+Vqm3IlQvAFFjVN1YYIdYhbLjAPdkgeVsWfcLDforHn6rR3QBZYZIlSBQSKRMY/tywTxeTCvK2zWcS0kbbFPtBcVth7VFFVPAZXhPi9yy1AvnldO6n7KLiupVmojlEMtv4FQkk604nal+j/dOplTATV8a9AJBbPRBZ/yQg57EG2Y2MRiHOQifJx0S5VbNyMm9bkS8TD7Goi59aCW6OT1gyeotWwLg60JRZTfyJ7lYWBSOzh0OnaCytRpSWtNZ6barPUeOnftbnJtE8rFhF7M4F66et0LI/cuvXYecwVwykovEVBKRF4HOK9GgSm17mQMtzrD7c558TbaucOWabYR04uhdAc3s10MkuONWG0wIQhgIChYVAGnFLvSpp2/aQEq3xrRSETxsixUIjsZyWWROkuA0IFnc8d7AmcnUBvRW7FT/5thWyk5agdYUGZ+7C1o69ihR1YxmoGh69fLMPIEOhYh572+3ckgl2SaV4uo9Gvkz8MMGRBcMIMlRirSwhCfozV2RyT5Wn1NgPpyc8zJL7QdOhL7Qxb+5WjnCVrQYHI2cCAwEAAQ==" + funding_account_id="2e181bc5ec273f2385439eecfacf967525c9d61003cc7c178ba4eaad84d1ac72" + funding_fs_port="9091" + port_forward_cmd="kubectl -n dev-wallet-testnet port-forward svc/full-service 9091:9090" + ;; + main) + export MC_FOG_REPORT_URL="fog://fog.prod.mobilecoinww.com" + export MC_WALLET_FILE=${MC_WALLET_FILE:-${GIT_BASE}/.mob/mainnet_secret_mnemonic.json} + export MC_FOG_AUTHORITY_SPKI="MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyr/99fvxi104MLgDgvWPVt01TuTJ+rN4qcNBUbF5i3EMM5zDZlugFHKPYPv7flCh5yDDYyLQHfWkxPQqCBAqlhSrCakvQH3HqDSpbM5FJg7pt0k5w+UQGWvP079iSEO5fMRhjE/lORkvk3/UKr2yIXjZ19iEgP8hlhk9xkI42DSg0iIhk59k3wEYPMGSkVarqlPoKBzx2+11CieXnbCkRvoNwLvdzLceY8QNoLc6h2/nht4bcjDCdB0MKNSKFLVp6XNHkVF66jC7QWTZRA/d4pgI5xa+GmkQ90zDZC2sBc+xfquVIVtk0nEvqSkUDZjv7AcJaq/VdPu4uj773ojrZz094PI4Q6sdbg7mfWrcq3ZQG8t9RDXD+6cgugCTFx2Cq/vJhDAPbQHmCEaMoXv2sRSfOhRjtMP1KmKUw5zXmAZa7s88+e7UXRQC+SS77V8s3hinE/I5Gqa/lzl73smhXx8l4CwGnXzlQ5h1lgEHnYLRFnIenNw/mdMGKlWH5HwHLX3hIujERCPAnGLDt+4MjcUiU0spDH3hC9mjPVA3ltaA3+Mk2lDw0kLrZ4Gv3/Ik9WPlYetOuWteMkR1fz6VOc13+WoTJPz0dVrJsK2bUz+YvdBsoHQBbUpCkmnQ5Ok+yiuWa5vYikEJ24SEr8wUiZ4Oe12KVEcjyDIxp6QoE8kCAwEAAQ==" + funding_account_id="ea7d2628e31ff7f546b193258e4b99f026521a99c2a5fdb76ac45258405f5b12" + funding_fs_port="9092" + port_forward_cmd="kubectl -n dev-wallet-mainnet port-forward svc/full-service 9092:9090" + ;; + *) + echo "ERROR: must be main or test" + usage + exit 1 + ;; +esac + +# Override the URL and account_id for the funding full-service if you want to use a different one +# Parse to get the host and port +TARGET_FS_URL=${TARGET_FS_URL:-"http://localhost:9090/wallet/v2"} +declare -A TARGET_FS_URL_PARSED +parse_url "${TARGET_FS_URL}" TARGET_FS_URL_PARSED + +# Set funding url and parse hostname/port +FUNDING_FS_URL=${FUNDING_FS_URL:-"http://localhost:${funding_fs_port}/wallet/v2"} +declare -A FUNDING_FS_URL_PARSED +parse_url "${FUNDING_FS_URL}" FUNDING_FS_URL_PARSED + +FUNDING_ACCOUNT_ID=${FUNDING_ACCOUNT_ID:-${funding_account_id}} +test_success=0 + +funding_mob() +{ + MC_FULL_SERVICE_PORT=${FUNDING_FS_URL_PARSED[port]} MC_FULL_SERVICE_HOST="${FUNDING_FS_URL_PARSED[proto]}${FUNDING_FS_URL_PARSED[host]}" mob "$@" +} + +target_mob() +{ + MC_FULL_SERVICE_PORT=${TARGET_FS_URL_PARSED[port]} MC_FULL_SERVICE_HOST="${TARGET_FS_URL_PARSED[proto]}${TARGET_FS_URL_PARSED[host]}" mob "$@" +} + +# Check to see if local full-service is running +if ! curl --connect-timeout 2 -s -f "${TARGET_FS_URL}" >/dev/null +then + echo "ERROR: Full-Service is not running on ${TARGET_FS_URL}" + echo " Please ensure the service is running and accessible." + exit 1 +else + echo "INFO: Connected successfully to local Full-Service at ${TARGET_FS_URL}" +fi + +# Check if the Funding Full-Service is running and accessible +read -r -d '' get_account_status_request </dev/null +then + echo "INFO: poetry not found, installing..." + pip install poetry +fi + +pushd "${GIT_BASE:?}/python" > /dev/null + +echo "INFO: Install mob(lecoin) package" +if ! command -v mob >/dev/null +then + echo "INFO: mob not found, installing..." + export PATH="${HOME}/.local/bin:${PATH}" + pip install . +fi + +echo "INFO: install test dependencies" +poetry install + +echo "INFO: Create wallet" +if target_mob list | grep test-wallet +then + echo: "ERROR: test-wallet already exists" + exit 1 +fi +target_mob -y create --name test-wallet + +echo "INFO: Get wallet address" +local_address=$(target_mob address list test-wallet -i 0) +echo "INFO: ${local_address}" + +echo "INFO: fund test wallet with 0.02 MOB" +funding_mob -y send "${funding_account_id}" 0.02 MOB "${local_address}" + +echo "INFO: wait for funds to arrive" +local_balance=$(target_mob balance test-wallet | cut -d' ' -f2) +while [[ "${local_balance}" == "0" ]] +do + local_balance=$(target_mob balance test-wallet | cut -d' ' -f2) + echo "INFO: balance: ${local_balance}" + sleep 2 +done + +echo "INFO: get funding account return address" +funding_account_address=$(funding_mob address list "${funding_account_id}" -i 0) +echo "INFO: ${funding_account_address}" + +echo "INFO: export wallet" +target_mob -y export test-wallet --file "${MC_WALLET_FILE}" + +echo "INFO: remove wallet so we can test import from file" +target_mob -y remove test-wallet + +echo "INFO: run tests" +if MC_FULL_SERVICE_PORT=${TARGET_FS_URL_PARSED[port]} MC_FULL_SERVICE_HOST="${TARGET_FS_URL_PARSED[proto]}${TARGET_FS_URL_PARSED[host]}" poetry run pytest -v +then + echo "INFO: tests passed" + test_success=1 +else + # this is a bit silly, but we want to catch failure and finish the script to drain any remaining funds + # we could trap exits, but there are lots of places where we could exit but not have things set up to + # drain funds + echo "ERROR: tests failed" +fi + +echo "INFO: look for leftover funds" +balances=$(target_mob balance) +while read -r b +do + # split out ammount from address and token type + counter=0 + account=$(echo "${b}" | cut -d' ' -f1) + amount=$(echo "${b}" | cut -d' ' -f2) + token=$(echo "${b}" | cut -d' ' -f3) + if (( $(echo "${amount} 0" | awk '{print ($1 > $2)}') )) + then + echo "INFO: found leftover funds: ${b}" + echo "INFO: drain funds to ${funding_account_address:0:5}...${funding_account_address: -5}" + echo "mob -y send ${account} all ${token} ..." + # I think there can be a timing issue when sending funds. If full-service has not yet polled the network, it + # might think its in sync, but might have transactions pending. As a work around, we will retry sending. + while ! target_mob -y send "${account}" all "${token}" "${funding_account_address}" + do + sleep 5 + echo "INFO: retrying send" + counter=$((counter+1)) + if ((counter > 5)) + then + echo "ERROR: out of retries - failed to send leftover funds back to funding account" + exit 1 + fi + done + fi +done <<< "${balances}" + +echo "INFO: remove wallet file" +rm "${MC_WALLET_FILE}" + +popd > /dev/null + +if ((test_success == 0)) +then + echo "ERROR: tests failed" + exit 1 +fi