From 6498ba33577b932c713a4c51d82a5dc51b40a98e Mon Sep 17 00:00:00 2001 From: "Peter St. John" Date: Thu, 7 Nov 2024 11:14:17 -0800 Subject: [PATCH] update notebook file --- .../core/data/test_load_notebook.ipynb | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/sub-packages/bionemo-core/tests/bionemo/core/data/test_load_notebook.ipynb b/sub-packages/bionemo-core/tests/bionemo/core/data/test_load_notebook.ipynb index 0fd318b031..7d61836dfc 100644 --- a/sub-packages/bionemo-core/tests/bionemo/core/data/test_load_notebook.ipynb +++ b/sub-packages/bionemo-core/tests/bionemo/core/data/test_load_notebook.ipynb @@ -2,14 +2,14 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from pathlib import Path\n", "import tempfile\n", "\n", - "from bionemo.testing.data.load import load" + "from bionemo.core.data.load import load" ] }, { @@ -21,34 +21,23 @@ "name": "stderr", "output_type": "stream", "text": [ - "Downloading data from 'nvidia/clara/esm2_pretrain_nemo2_testdata:1.0' to file '/tmp/tmpglnde0ml/dc23f4aaad387ecc12e53d56b8176430-esm2_pretrain_nemo2_testdata:1.0'.\n" + "Downloading data from 'nvidia/clara/esm2_pretrain_nemo2_testdata:1.0' to file '/tmp/tmp_v_0_64q/dc23f4aaad387ecc12e53d56b8176430-esm2_pretrain_nemo2_testdata:1.0'.\n", + "Untarring contents of '/tmp/tmp_v_0_64q/dc23f4aaad387ecc12e53d56b8176430-esm2_pretrain_nemo2_testdata:1.0' to '/tmp/tmp_v_0_64q/dc23f4aaad387ecc12e53d56b8176430-esm2_pretrain_nemo2_testdata:1.0.untar'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Getting files to download...\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "asyncio.run() cannot be called from a running event loop", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[2], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# xfail -- we need to file a bug or figure out how to call the ngcsdk from a jupyter notebook\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# NBVAL_RAISES_EXCEPTION\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tempfile\u001b[38;5;241m.\u001b[39mTemporaryDirectory() \u001b[38;5;28;01mas\u001b[39;00m cache_dir:\n\u001b[0;32m----> 4\u001b[0m \u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mesm2/testdata_esm2_pretrain:2.0\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msource\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mngc\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mPath\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/workspaces/bionemo-framework/sub-packages/bionemo-testing/src/bionemo/testing/data/load.py:158\u001b[0m, in \u001b[0;36mload\u001b[0;34m(model_or_data_tag, source, resources, cache_dir)\u001b[0m\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 156\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSource \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msource\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m not supported.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 158\u001b[0m download \u001b[38;5;241m=\u001b[39m \u001b[43mpooch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mretrieve\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 159\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 160\u001b[0m \u001b[43m \u001b[49m\u001b[43mknown_hash\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresource\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msha256\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 161\u001b[0m \u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 162\u001b[0m \u001b[43m \u001b[49m\u001b[43mdownloader\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_fn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 163\u001b[0m \u001b[43m \u001b[49m\u001b[43mprocessor\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprocessor\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 164\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 166\u001b[0m \u001b[38;5;66;03m# Pooch by default returns a list of unpacked files if they unpack a zipped or tarred directory. Instead of that, we\u001b[39;00m\n\u001b[1;32m 167\u001b[0m \u001b[38;5;66;03m# just want the unpacked, parent folder.\u001b[39;00m\n\u001b[1;32m 168\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(download, \u001b[38;5;28mlist\u001b[39m):\n", - "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/pooch/core.py:239\u001b[0m, in \u001b[0;36mretrieve\u001b[0;34m(url, known_hash, fname, path, processor, downloader, progressbar)\u001b[0m\n\u001b[1;32m 236\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m downloader \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 237\u001b[0m downloader \u001b[38;5;241m=\u001b[39m choose_downloader(url, progressbar\u001b[38;5;241m=\u001b[39mprogressbar)\n\u001b[0;32m--> 239\u001b[0m \u001b[43mstream_download\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfull_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mknown_hash\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdownloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpooch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 241\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m known_hash \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 242\u001b[0m get_logger()\u001b[38;5;241m.\u001b[39minfo(\n\u001b[1;32m 243\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSHA256 hash of downloaded file: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 244\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUse this value as the \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mknown_hash\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m argument of \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpooch.retrieve\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 247\u001b[0m file_hash(\u001b[38;5;28mstr\u001b[39m(full_path)),\n\u001b[1;32m 248\u001b[0m )\n", - "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/pooch/core.py:807\u001b[0m, in \u001b[0;36mstream_download\u001b[0;34m(url, fname, known_hash, downloader, pooch, retry_if_failed)\u001b[0m\n\u001b[1;32m 803\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 804\u001b[0m \u001b[38;5;66;03m# Stream the file to a temporary so that we can safely check its\u001b[39;00m\n\u001b[1;32m 805\u001b[0m \u001b[38;5;66;03m# hash before overwriting the original.\u001b[39;00m\n\u001b[1;32m 806\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m temporary_file(path\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mstr\u001b[39m(fname\u001b[38;5;241m.\u001b[39mparent)) \u001b[38;5;28;01mas\u001b[39;00m tmp:\n\u001b[0;32m--> 807\u001b[0m \u001b[43mdownloader\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtmp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpooch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 808\u001b[0m hash_matches(tmp, known_hash, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, source\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mstr\u001b[39m(fname\u001b[38;5;241m.\u001b[39mname))\n\u001b[1;32m 809\u001b[0m shutil\u001b[38;5;241m.\u001b[39mmove(tmp, \u001b[38;5;28mstr\u001b[39m(fname))\n", - "File \u001b[0;32m/workspaces/bionemo-framework/sub-packages/bionemo-testing/src/bionemo/testing/data/load.py:98\u001b[0m, in \u001b[0;36mNGCDownloader.__call__\u001b[0;34m(self, url, output_file, _)\u001b[0m\n\u001b[1;32m 95\u001b[0m ngc_dirname \u001b[38;5;241m=\u001b[39m Path(url)\u001b[38;5;241m.\u001b[39mname\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_v\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 97\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tempfile\u001b[38;5;241m.\u001b[39mTemporaryDirectory(\u001b[38;5;28mdir\u001b[39m\u001b[38;5;241m=\u001b[39moutput_file\u001b[38;5;241m.\u001b[39mparent) \u001b[38;5;28;01mas\u001b[39;00m temp_dir:\n\u001b[0;32m---> 98\u001b[0m \u001b[43mdownload_fns\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mngc_registry\u001b[49m\u001b[43m]\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemp_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfile_patterns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfilename\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 99\u001b[0m shutil\u001b[38;5;241m.\u001b[39mmove(Path(temp_dir) \u001b[38;5;241m/\u001b[39m ngc_dirname \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfilename, output_file)\n", - "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/ngcbpc/util/utils.py:540\u001b[0m, in \u001b[0;36mextra_args..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 538\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdebug\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 539\u001b[0m warnings\u001b[38;5;241m.\u001b[39mwarn(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdebug\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m argument is deprecated and will be removed soon.\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mDeprecationWarning\u001b[39;00m)\n\u001b[0;32m--> 540\u001b[0m retval \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 541\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m retval\n", - "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/registry/api/resources.py:143\u001b[0m, in \u001b[0;36mResourceAPI.download_version\u001b[0;34m(self, target, destination, file_patterns, exclude_patterns)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ResourceNotFoundException:\n\u001b[1;32m 141\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ResourceNotFoundException(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtarget\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m could not be found.\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 143\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_download\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmrt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdownload_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfile_patterns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfile_patterns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexclude_patterns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexclude_patterns\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/registry/api/resources.py:999\u001b[0m, in \u001b[0;36mResourceAPI._download\u001b[0;34m(self, mrt, download_dir, file_patterns, exclude_patterns)\u001b[0m\n\u001b[1;32m 990\u001b[0m download_dir \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdownload_dir\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmrt\u001b[38;5;241m.\u001b[39mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_v\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmrt\u001b[38;5;241m.\u001b[39mversion\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 991\u001b[0m started_at \u001b[38;5;241m=\u001b[39m datetime\u001b[38;5;241m.\u001b[39mdatetime\u001b[38;5;241m.\u001b[39mnow()\n\u001b[1;32m 992\u001b[0m (\n\u001b[1;32m 993\u001b[0m elapsed,\n\u001b[1;32m 994\u001b[0m download_count,\n\u001b[1;32m 995\u001b[0m download_size,\n\u001b[1;32m 996\u001b[0m failed_count,\n\u001b[1;32m 997\u001b[0m _,\n\u001b[1;32m 998\u001b[0m _,\n\u001b[0;32m--> 999\u001b[0m ) \u001b[38;5;241m=\u001b[39m \u001b[43masync_download\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdirect_download_files\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1000\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresource\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1001\u001b[0m \u001b[43m \u001b[49m\u001b[43mmrt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1002\u001b[0m \u001b[43m \u001b[49m\u001b[43mmrt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43morg\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1003\u001b[0m \u001b[43m \u001b[49m\u001b[43mmrt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mteam\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1004\u001b[0m \u001b[43m \u001b[49m\u001b[43mmrt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mversion\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1005\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1006\u001b[0m \u001b[43m \u001b[49m\u001b[43mpaginated\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1007\u001b[0m \u001b[43m \u001b[49m\u001b[43mdl_files_with_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1008\u001b[0m \u001b[43m \u001b[49m\u001b[43mtotal_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1009\u001b[0m \u001b[43m \u001b[49m\u001b[43mdownload_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1010\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1011\u001b[0m ended_at \u001b[38;5;241m=\u001b[39m datetime\u001b[38;5;241m.\u001b[39mdatetime\u001b[38;5;241m.\u001b[39mnow()\n\u001b[1;32m 1012\u001b[0m status \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFAILED\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m failed_count \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCOMPLETED\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", - "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/ngcbpc/transfer/async_download.py:932\u001b[0m, in \u001b[0;36mdirect_download_files\u001b[0;34m(dl_type, name, org, team, version, url, paginated, dl_files_with_size, size, download_dir)\u001b[0m\n\u001b[1;32m 930\u001b[0m time_started \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mmonotonic()\n\u001b[1;32m 931\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 932\u001b[0m \u001b[43masyncio\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 933\u001b[0m \u001b[43m \u001b[49m\u001b[43m_direct_download_files\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 934\u001b[0m \u001b[43m \u001b[49m\u001b[43mdl_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 935\u001b[0m \u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 936\u001b[0m \u001b[43m \u001b[49m\u001b[43morg\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 937\u001b[0m \u001b[43m \u001b[49m\u001b[43mteam\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 938\u001b[0m \u001b[43m \u001b[49m\u001b[43mversion\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 939\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 940\u001b[0m \u001b[43m \u001b[49m\u001b[43mpaginated\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 941\u001b[0m \u001b[43m \u001b[49m\u001b[43mdownload_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 942\u001b[0m \u001b[43m \u001b[49m\u001b[43mauth_org\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morg\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 943\u001b[0m \u001b[43m \u001b[49m\u001b[43mauth_team\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mteam\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 944\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 945\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 946\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[1;32m 947\u001b[0m printer\u001b[38;5;241m.\u001b[39mprint_ok(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mTerminating download...\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m/usr/lib/python3.10/asyncio/runners.py:33\u001b[0m, in \u001b[0;36mrun\u001b[0;34m(main, debug)\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Execute the coroutine and return the result.\u001b[39;00m\n\u001b[1;32m 10\u001b[0m \n\u001b[1;32m 11\u001b[0m \u001b[38;5;124;03mThis function runs the passed coroutine, taking care of\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[38;5;124;03m asyncio.run(main())\u001b[39;00m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 32\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m events\u001b[38;5;241m.\u001b[39m_get_running_loop() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 33\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 34\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masyncio.run() cannot be called from a running event loop\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 36\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m coroutines\u001b[38;5;241m.\u001b[39miscoroutine(main):\n\u001b[1;32m 37\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ma coroutine was expected, got \u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(main))\n", - "\u001b[0;31mRuntimeError\u001b[0m: asyncio.run() cannot be called from a running event loop" + "{\n", + " \"download_end\": \"2024-11-07 19:13:48\",\n", + " \"download_start\": \"2024-11-07 19:13:46\",\n", + " \"download_time\": \"2s\",\n", + " \"files_downloaded\": 1,\n", + " \"local_path\": \"/tmp/tmp_v_0_64q/tmpgadmjb1k/esm2_pretrain_nemo2_testdata_v1.0\",\n", + " \"size_downloaded\": \"69.91 MB\",\n", + " \"status\": \"COMPLETED\"\n", + "}\n" ] } ], @@ -61,16 +50,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Downloading data from 's3://general-purpose/esm2/pretrain/2024_03_sanity.tar.gz' to file '/tmp/tmpzm37lbjr/f796e1ca28311606ff7dd62a067508bf-2024_03_sanity.tar.gz'.\n", - "s3://general-purpose/esm2/pretrain/2024_03_sanity.tar.gz: 100%|██████████| 73.3M/73.3M [00:03<00:00, 23.0MB/s]\n", - "Untarring contents of '/tmp/tmpzm37lbjr/f796e1ca28311606ff7dd62a067508bf-2024_03_sanity.tar.gz' to '/tmp/tmpzm37lbjr/f796e1ca28311606ff7dd62a067508bf-2024_03_sanity.tar.gz.untar'\n" + "Downloading data from 's3://general-purpose/esm2/pretrain/2024_03_sanity.tar.gz' to file '/tmp/tmpjnvk7m8k/f796e1ca28311606ff7dd62a067508bf-2024_03_sanity.tar.gz'.\n", + "s3://general-purpose/esm2/pretrain/2024_03_sanity.tar.gz: 100%|██████████| 73.3M/73.3M [00:01<00:00, 38.7MB/s]\n", + "Untarring contents of '/tmp/tmpjnvk7m8k/f796e1ca28311606ff7dd62a067508bf-2024_03_sanity.tar.gz' to '/tmp/tmpjnvk7m8k/f796e1ca28311606ff7dd62a067508bf-2024_03_sanity.tar.gz.untar'\n" ] } ],