zip_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import zipfile\n",
    "from tqdm import tqdm\n",
    "from data import Modality\n",
    "\n",
    "def zipdir(path, ziph):\n",
    "    # ziph is zipfile handle\n",
    "    for root, dirs, files in os.walk(path):\n",
    "        for file in tqdm(files):\n",
    "            ziph.write(os.path.join(root, file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/25 [00:00<?, ?it/s]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "  4%|▍         | 1/25 [16:06<6:26:42, 966.77s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "  8%|▊         | 2/25 [34:59<6:29:43, 1016.67s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 12%|█▏        | 3/25 [49:03<5:53:42, 964.67s/it] \n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 16%|█▌        | 4/25 [53:14<4:22:43, 750.66s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 20%|██        | 5/25 [1:00:48<3:40:32, 661.64s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 24%|██▍       | 6/25 [1:10:37<3:22:36, 639.80s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 28%|██▊       | 7/25 [1:15:47<2:42:16, 540.89s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 32%|███▏      | 8/25 [1:43:15<4:07:24, 873.21s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 36%|███▌      | 9/25 [1:52:49<3:28:50, 783.15s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 40%|████      | 10/25 [2:04:44<3:10:40, 762.73s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 44%|████▍     | 11/25 [2:28:06<3:42:46, 954.74s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 48%|████▊     | 12/25 [2:46:49<3:37:48, 1005.23s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 52%|█████▏    | 13/25 [2:55:11<2:50:48, 854.03s/it] \n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 56%|█████▌    | 14/25 [2:58:43<2:01:15, 661.43s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 60%|██████    | 15/25 [3:02:28<1:28:27, 530.71s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 64%|██████▍   | 16/25 [3:03:09<57:33, 383.74s/it]  \n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 68%|██████▊   | 17/25 [3:04:41<39:29, 296.19s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 72%|███████▏  | 18/25 [3:07:46<30:40, 262.88s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 76%|███████▌  | 19/25 [3:10:13<22:47, 227.96s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 80%|████████  | 20/25 [3:11:35<15:21, 184.37s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 84%|████████▍ | 21/25 [3:12:45<09:59, 149.82s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 88%|████████▊ | 22/25 [3:13:33<05:58, 119.38s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 92%|█████████▏| 23/25 [3:16:01<04:15, 127.94s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      " 96%|█████████▌| 24/25 [3:18:53<02:21, 141.31s/it]\n",
      "0it [00:00, ?it/s]\u001b[A"
     ]
    }
   ],
   "source": [
    "basePath = 'pats/data'\n",
    "common_files = ['cmu_intervals_df.csv', 'cmu_intervals_df_transforms.csv', 'missing_intervals.h5']\n",
    "#speaker = 'lec_cosmic'\n",
    "for speaker in tqdm(Modality('pats/data').speakers):\n",
    "  with zipfile.ZipFile(os.path.join(basePath,'zips/{}.zip'.format(speaker)), 'w', zipfile.ZIP_DEFLATED) as zipf:\n",
    "    zipdir(os.path.join(basePath, 'processed/{}/'.format(speaker)), zipf)\n",
    "    for f in common_files:\n",
    "      zipf.write(os.path.join(basePath, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]\n",
      "100%|██████████| 2107/2107 [07:20<00:00,  3.88it/s]\n",
      "100%|██████████| 4083/4083 [07:22<00:00,  8.76it/s]\n",
      "100%|██████████| 470/470 [01:01<00:00,  8.18it/s]\n",
      "100%|██████████| 4636/4636 [08:39<00:00,  8.32it/s]\n",
      "100%|██████████| 313/313 [00:53<00:00,  5.82it/s]\n",
      "100%|██████████| 1201/1201 [02:26<00:00,  8.27it/s]\n",
      "100%|██████████| 84/84 [00:31<00:00,  1.61it/s]\n",
      "100%|██████████| 1170/1170 [02:44<00:00,  6.21it/s]\n",
      "100%|██████████| 4408/4408 [16:48<00:00,  3.68it/s]\n",
      "100%|██████████| 4633/4633 [16:08<00:00,  4.67it/s]\n",
      "100%|██████████| 4367/4367 [19:55<00:00,  4.49it/s]\n",
      "100%|██████████| 167/167 [00:33<00:00,  4.13it/s]\n",
      " 28%|██▊       | 995/3582 [02:40<08:42,  4.95it/s]"
     ]
    }
   ],
   "source": [
    "basePath = 'pats/data'\n",
    "with zipfile.ZipFile(os.path.join(basePath,'zips/{}.zip'.format('all')), 'w', zipfile.ZIP_DEFLATED) as zipf:\n",
    "  zipdir(os.path.join(basePath, 'processed'), zipf)\n",
    "  for f in common_files:\n",
    "    zipf.write(os.path.join(basePath, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}