get_promoters.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d32ae2b1-9d32-4902-940a-5737f1723ab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "221c3841-d4b3-424c-a62a-f7ca10e78c4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import genomepy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "13ad1032-39ee-4b95-8cce-f6fc0768d949",
   "metadata": {},
   "outputs": [],
   "source": [
    "from Bio.Seq import Seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "47d5a60b-ee24-42c3-8f2d-9276923f6a6b",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m15:15:31\u001b[0m \u001b[1m|\u001b[0m \u001b[34mINFO\u001b[0m \u001b[1m|\u001b[0m Downloading assembly summaries from UCSC\n",
      "\u001b[32m15:15:32\u001b[0m \u001b[1m|\u001b[0m \u001b[34mINFO\u001b[0m \u001b[1m|\u001b[0m Downloading genome from UCSC. Target URL: https://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/hg38.fa.gz...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "692d0f2718594f00bbd907ca2b29ca15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Download:   0%|          | 0.00/938M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m15:17:02\u001b[0m \u001b[1m|\u001b[0m \u001b[34mINFO\u001b[0m \u001b[1m|\u001b[0m Genome download successful, starting post processing...\n",
      "\u001b[32m15:17:16\u001b[0m \u001b[1m|\u001b[0m \u001b[34mINFO\u001b[0m \u001b[1m|\u001b[0m name: hg38\n",
      "\u001b[32m15:17:16\u001b[0m \u001b[1m|\u001b[0m \u001b[34mINFO\u001b[0m \u001b[1m|\u001b[0m local name: hg38\n",
      "\u001b[32m15:17:16\u001b[0m \u001b[1m|\u001b[0m \u001b[34mINFO\u001b[0m \u001b[1m|\u001b[0m fasta: /Users/garyk/Documents/code/engreitz_lab/get_promoters/genomes/hg38/hg38.fa\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47da2e6d735c4300b63ebb6cb36b7b15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filtering Fasta: 0.00 lines [00:00, ? lines/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m15:18:59\u001b[0m \u001b[1m|\u001b[0m \u001b[34mINFO\u001b[0m \u001b[1m|\u001b[0m Downloading the ncbiRefSeq annotation from the UCSC MySQL database.\n"
     ]
    },
    {
     "ename": "GenomeDownloadError",
     "evalue": "An error occured while installing the gene annotation for hg38 from UCSC.\nIf you think the annotation should be there, please file a bug report at: https://github.com/vanheeringen-lab/genomepy/issues\n\nError: Installing gene annotations requires missing UCSC tools. See https://github.com/vanheeringen-lab/genomepy#pip for download links and details.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/jup/lib/python3.9/site-packages/genomepy/providers/ucsc.py:343\u001b[0m, in \u001b[0;36mUcscProvider.download_annotation\u001b[0;34m(self, name, genomes_dir, localname, **kwargs)\u001b[0m\n\u001b[1;32m    342\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 343\u001b[0m     \u001b[43mdownload_annotation\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mannot\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgenomes_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlocalname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    344\u001b[0m     logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAnnotation download successful\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/miniconda3/envs/jup/lib/python3.9/site-packages/genomepy/providers/ucsc.py:549\u001b[0m, in \u001b[0;36mdownload_annotation\u001b[0;34m(name, annot, genomes_dir, localname, n)\u001b[0m\n\u001b[1;32m    545\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    546\u001b[0m \u001b[38;5;124;03mDownload the extended genePred file from the UCSC MySQL database.\u001b[39;00m\n\u001b[1;32m    547\u001b[0m \u001b[38;5;124;03mNext convert this to a BED and GTF file.\u001b[39;00m\n\u001b[1;32m    548\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 549\u001b[0m \u001b[43mcheck_ucsc_tools\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    551\u001b[0m out_dir \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(genomes_dir, localname)\n",
      "File \u001b[0;32m~/miniconda3/envs/jup/lib/python3.9/site-packages/genomepy/utils.py:179\u001b[0m, in \u001b[0;36mcheck_ucsc_tools\u001b[0;34m(tools)\u001b[0m\n\u001b[1;32m    178\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m shutil\u001b[38;5;241m.\u001b[39mwhich(tool) \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 179\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\n\u001b[1;32m    180\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInstalling gene annotations requires missing UCSC tools. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    181\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSee https://github.com/vanheeringen-lab/genomepy#pip \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    182\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfor download links and details.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    183\u001b[0m     )\n",
      "\u001b[0;31mOSError\u001b[0m: Installing gene annotations requires missing UCSC tools. See https://github.com/vanheeringen-lab/genomepy#pip for download links and details.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mGenomeDownloadError\u001b[0m                       Traceback (most recent call last)",
      "Input \u001b[0;32mIn [15]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mgenomepy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minstall_genome\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhg38\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mannotation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprovider\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mUCSC\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgenomes_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m./genomes\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/jup/lib/python3.9/site-packages/genomepy/functions.py:264\u001b[0m, in \u001b[0;36minstall_genome\u001b[0;34m(name, provider, genomes_dir, localname, mask, keep_alt, regex, invert_match, bgzip, annotation, only_annotation, skip_matching, skip_filter, threads, force, **kwargs)\u001b[0m\n\u001b[1;32m    262\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m force:\n\u001b[1;32m    263\u001b[0m         _delete_extensions(out_dir, [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mannotation.gtf\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mannotation.bed\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[0;32m--> 264\u001b[0m     \u001b[43mprovider\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload_annotation\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgenomes_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlocalname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocalname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    265\u001b[0m     annotation_downloaded \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mbool\u001b[39m(\n\u001b[1;32m    266\u001b[0m         glob_ext_files(out_dir, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mannotation.gtf\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    267\u001b[0m     ) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mbool\u001b[39m(glob_ext_files(out_dir, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mannotation.bed\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n\u001b[1;32m    269\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m annotation_downloaded:\n",
      "File \u001b[0;32m~/miniconda3/envs/jup/lib/python3.9/site-packages/genomepy/providers/ucsc.py:346\u001b[0m, in \u001b[0;36mUcscProvider.download_annotation\u001b[0;34m(self, name, genomes_dir, localname, **kwargs)\u001b[0m\n\u001b[1;32m    344\u001b[0m     logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAnnotation download successful\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    345\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 346\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m GenomeDownloadError(\n\u001b[1;32m    347\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn error occured while installing the gene annotation for \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m from \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    348\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIf you think the annotation should be there, please file a bug report at: \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    349\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://github.com/vanheeringen-lab/genomepy/issues\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    350\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;241m.\u001b[39margs[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    351\u001b[0m     )\n\u001b[1;32m    353\u001b[0m \u001b[38;5;66;03m# Add annotation URL to readme\u001b[39;00m\n\u001b[1;32m    354\u001b[0m readme \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(genomes_dir, localname, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mREADME.txt\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mGenomeDownloadError\u001b[0m: An error occured while installing the gene annotation for hg38 from UCSC.\nIf you think the annotation should be there, please file a bug report at: https://github.com/vanheeringen-lab/genomepy/issues\n\nError: Installing gene annotations requires missing UCSC tools. See https://github.com/vanheeringen-lab/genomepy#pip for download links and details."
     ]
    }
   ],
   "source": [
    "genomepy.install_genome(\"hg38\", annotation=True, provider=\"UCSC\", genomes_dir=\"./genomes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b2db508a-fbc0-46ad-b1f9-1d9031761adb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# path = '/oak/stanford/groups/engreitz/Users/gbrixi/promoters/data'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "8fbf0ca6-40d4-459f-b975-0344bae3046b",
   "metadata": {},
   "outputs": [],
   "source": [
    "tss = pd.read_csv('./data/RefSeqCurated.170308.bed.CollapsedGeneBounds.hg38.TSS500bp.bed', sep='\\t', header = None)\n",
    "\n",
    "seqs = []\n",
    "for _,row in tss.iterrows():\n",
    "    if '_' in row[0]:\n",
    "        row[0] = row[0].split('_')[0]\n",
    "    seqs.append(g[row[0]][row[1]:row[2]].seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "5f474bb5-00b3-4b8e-ba8d-7f286de33ac0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "g = genomepy.Genome(\"hg38\", genomes_dir=\"./genomes/\")\n",
    "\n",
    "directed_strand = []\n",
    "\n",
    "for i in range(len(seqs)):\n",
    "    strandedness = tss.iloc[i][5]\n",
    "    \n",
    "    if strandedness == '-':\n",
    "        rev = Seq(seqs[i]).reverse_complement()\n",
    "        directed_strand.append(str(rev))\n",
    "    else:\n",
    "        directed_strand.append(seqs[i])\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "3e3e816e-ac36-4f2a-a92a-3ac57993bedd",
   "metadata": {},
   "outputs": [],
   "source": [
    "rev = Seq(directed_strand[1]).reverse_complement()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "1209434e-d63a-4d63-93bb-0fb1e9ada96a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tss['+_seq'] = seqs\n",
    "tss['directed_seq'] = directed_strand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "1c4e0c80-e62d-4b8b-a657-d3ea65060370",
   "metadata": {},
   "outputs": [],
   "source": [
    "tss.to_csv('./data/promoter_500bp_tss.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80410c23-548b-4465-ac95-05da4fdcd9a5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}