test_s5cmdpy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e8bdb712",
   "metadata": {},
   "source": [
    "use directly:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b154f220",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ca488d5a8f74076818204f346d7bd8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Listing S3 objects: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dataset.json': (181153676, '2024/06/22 04:14:52')}\n"
     ]
    }
   ],
   "source": [
    "import s5cmdpy as s5py\n",
    "print(s5py.ls(\"s3://dataset-artstation-uw2/\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef158c75",
   "metadata": {},
   "source": [
    "use with runner:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "225443b5-109c-4d95-b558-4fdaf19dc314",
   "metadata": {},
   "outputs": [],
   "source": [
    "from s5cmdpy import S5CmdRunner\n",
    "runner = S5CmdRunner()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "98e684cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8af299361aab4f76b9bc6df9d2979c3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Listing S3 objects: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dataset.json': (181153676, '2024/06/22 04:14:52')}\n"
     ]
    }
   ],
   "source": [
    "print(runner.ls(\"s3://dataset-artstation-uw2/\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f4f45db7-f5f3-461e-a82c-c337875ab3e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dca27e1ca784570b10c8f390b3c4c79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Processing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# local_txt: `cp s3://dataset-artstation-uw2/artists/__andrey__/1841730##GZGgW.json .`\n",
    "local_txt_path = \"s5cmd_test.txt\"\n",
    "runner.run(local_txt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b12bfd82-ad7f-4bc1-be9e-036bda500dc9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cp s3://dataset-artstation-uw2/_dev/s5cmd_test.txt /tmp/s5cmd_test.txt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85592361b7154fe8be054d9e0959e14f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Processing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# (在sagemaker会用到, 或者用来留档方便复现); \n",
    "# 扩展s5cmd run something.txt 到支持传入s3上保存的txt文件\n",
    "txt_s3_uri = \"s3://dataset-artstation-uw2/_dev/s5cmd_test.txt\"\n",
    "\n",
    "runner = S5CmdRunner()\n",
    "runner.run(txt_s3_uri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0cbd3137-99c2-4721-920c-f4dcd2748e32",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "697998c893704ce3bfa3ac20f3034731",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[interval=5] running s5cmd:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 输入一系列要下载的s3 uri, 会创建s5cmd run需要的commands.txt, 然后调用s5cmd run <commands.txt>\n",
    "\n",
    "s3_uris = [\n",
    "    's3://dataset-artstation-uw2/artists/__andrey__/1841730##GZGgW.json', \n",
    "    's3://dataset-artstation-uw2/artists/__andrey__/2249992##q5Y22.json'\n",
    "          ]\n",
    "destination_dir = '/home/ubuntu/datasets/s5cmd_test'\n",
    "# runner.download_from_s3_list(s3_uris, destination_dir, simplified_print=False)\n",
    "runner.download_from_s3_list(s3_uris, destination_dir, simplified_print=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e73a2a56-eafa-4c7f-99dc-ca83ed25d43d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cp /tmp/ml_caformer_m36_dec-5-97527.onnx s3://dataset-artstation-uw2/_dev/ml_caformer_m36_dec-5-97527.onnx\n"
     ]
    }
   ],
   "source": [
    "# 输入一个url, 复制文件到s3\n",
    "\n",
    "target_url = \"https://huggingface.co/kiriyamaX/mld-caformer/resolve/main/ml_caformer_m36_dec-5-97527.onnx\"\n",
    "dst_s3_uri = \"s3://dataset-artstation-uw2/_dev/\"\n",
    "\n",
    "runner.cp(target_url, dst_s3_uri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "74b52b46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fbc509d92ad4b57a2d69878fd9c08fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Listing S3 objects: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 测试ls速度: 900k个文件, 用ub.traverses (botocore, list-object-v2) 需要02:24; s5cmd ls 用了01:06\n",
    "\n",
    "ls_dir = \"s3://unidataset-danbooru/upscaled_sub891289_2x_run2/\"\n",
    "files = runner.ls(ls_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d619e2f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'metadata': {'len': 899604,\n",
       "  'value_types': {'tuple': 899604},\n",
       "  'is_nested': False},\n",
       " 'preview': [('image_1.webp', (238712, '2023/10/18 19:37:06')),\n",
       "  ('image_10.webp', (119102, '2023/10/18 19:13:40')),\n",
       "  ('image_100.webp', (459380, '2023/10/18 19:14:10')),\n",
       "  ('image_1000.webp', (143000, '2023/10/18 19:13:53')),\n",
       "  ('image_10000.webp', (257036, '2023/10/18 19:12:24'))]}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import unibox as ub\n",
    "ub.peeks(files, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d43cd72",
   "metadata": {},
   "source": [
    "测试带s5cmd argument时:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "570b9ad6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-04-09 02:20:27,256 [INFO] UniLogger: S5CmdRunner.download_from_s3_list: Generated command file: /tmp/s5cmd_commands_20240409-022027_ea741dd0.txt\n",
      "2024-04-09 02:20:27,257 [INFO] UniLogger: S5CmdRunner.download_from_s3_list: Downloading 2 files from S3 to /home/ubuntu/datasets/s5cmd_test\n",
      "Incorrect Usage: flag provided but not defined: -part-size\n",
      "See 's5cmd --help' for usage\n",
      "2024-04-09 02:20:27,263 [INFO] UniLogger: S5CmdRunner.download_from_s3_list: Downloaded 2 files from S3 to /home/ubuntu/datasets/s5cmd_test\n",
      "2024-04-09 02:20:27,264 [INFO] UniLogger: S5CmdRunner.download_from_s3_list: removing command file: /tmp/s5cmd_commands_20240409-022027_ea741dd0.txt\n"
     ]
    }
   ],
   "source": [
    "# runner = S5CmdRunner(config_string=\"--numworkers=4 --part-size=64\")\n",
    "\n",
    "# s3_uris = [\n",
    "#     's3://dataset-artstation-uw2/artists/__andrey__/1841730##GZGgW.json', \n",
    "#     's3://dataset-artstation-uw2/artists/__andrey__/2249992##q5Y22.json'\n",
    "#           ]\n",
    "# destination_dir = '/home/ubuntu/datasets/s5cmd_test'\n",
    "\n",
    "# runner.download_from_s3_list(s3_uris, destination_dir, simplified_print=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}