notebooks/Download repositories.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting data\n",
    "\n",
    "Since we plan to analyze a few repositories in this workshop, let's download them.\n",
    "\n",
    "We'll first get metadata about a user or organization thanks to GitHub API, and then download the repositories that interest us the most."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieving metadata about a user/organization\n",
    "\n",
    "We iterate as long as the API gives us a pointer to another response page. We filter forks to focus on original repositories."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from logging import getLogger\n",
    "from pathlib import Path\n",
    "\n",
    "from coloredlogs import install as coloredlogs_install\n",
    "\n",
    "\n",
    "coloredlogs_install()\n",
    "logger = getLogger(\"downloader\")\n",
    "\n",
    "\n",
    "repos_dir = Path(\"repos\")\n",
    "git_data_dir = repos_dir / \"git-data\"\n",
    "git_data_dir.mkdir(parents=True, exist_ok=True)\n",
    "repos_json = repos_dir / \"repos.json\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To use GitHub API, we need a token. Normally it has been made available through the `GITHUB_TOKEN` environment variable. Let's check!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import environ\n",
    "TOKEN = environ.get(\"GITHUB_TOKEN\")\n",
    "if TOKEN is None:\n",
    "    logger.critical(\"Could not find GITHUB_TOKEN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from json import dump as json_dump\n",
    "from operator import itemgetter\n",
    "from re import search as re_search\n",
    "from typing import Any, Dict, List, Optional\n",
    "\n",
    "import requests\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "\n",
    "def list_repositories(user: str,\n",
    "                      token: str,\n",
    "                      max_size_mb: int,\n",
    "                      repos_number: int\n",
    "                     ) -> List[Dict[str, Any]]:\n",
    "\n",
    "    def parse_last(link_header: str) -> Optional[int]:\n",
    "        match = re_search(\n",
    "            r'<'\n",
    "            r'https://api.github.com/user/'\n",
    "            r'[^/]+/repos\\?[^>]*page='\n",
    "            r'(\\d+)'\n",
    "            r'[^>]*>; rel=\"last\"',\n",
    "            link_header)\n",
    "        if match is None:\n",
    "            return None\n",
    "        return int(match.group(1))\n",
    "\n",
    "    repos_list_headers = dict(Authorization=\"token %s\" % token)\n",
    "    repos_url = \"https://api.github.com/users/%s/repos\" % user\n",
    "\n",
    "    request_total = requests.get(repos_url,\n",
    "                                 headers=repos_list_headers)\n",
    "    total_pages = parse_last(request_total.headers[\"Link\"])\n",
    "    assert total_pages is not None\n",
    "\n",
    "    logger.info(\"Retrieving repos list for user %s\" % user)\n",
    "    repos = []\n",
    "    for page in tqdm(range(1, total_pages + 1)):\n",
    "        request = requests.get(\"%s?page=%d\" % (repos_url, page),\n",
    "                               headers=repos_list_headers)\n",
    "        request.raise_for_status()\n",
    "        for repo in request.json():\n",
    "            if repo[\"fork\"]:\n",
    "                continue\n",
    "            repos.append(dict(\n",
    "                name=repo[\"name\"],\n",
    "                branch=repo[\"default_branch\"],\n",
    "                clone_url=repo[\"clone_url\"],\n",
    "                size=repo[\"size\"],\n",
    "                stars=repo[\"stargazers_count\"]\n",
    "            ))\n",
    "\n",
    "    if max_size_mb is not None:\n",
    "        logger.info(\n",
    "            \"Filtering to keep only repositories under %.2f MB\",\n",
    "            max_size_mb\n",
    "        )\n",
    "        repos = [repo for repo in repos\n",
    "                 if repo[\"size\"] <= max_size_mb * 1024]\n",
    "\n",
    "    if repos_number is not None:\n",
    "        logger.info(\n",
    "            \"Filtering to keep only the %d most popular repositories\",\n",
    "            repos_number\n",
    "        )\n",
    "        repos = [repo for repo in sorted(repos,\n",
    "                                         key=itemgetter(\"stars\"),\n",
    "                                         reverse=True)][:repos_number]\n",
    "\n",
    "    def get_repo_sha_url(user: str, repo: str, branch: str):\n",
    "        return \"https://api.github.com/repos/%s/%s/commits/%s\" % (\n",
    "            user,\n",
    "            repo,\n",
    "            branch\n",
    "        )\n",
    "\n",
    "    logger.info(\"Getting SHA1 for each repository\")\n",
    "    repo_sha_headers = dict(\n",
    "        Authorization=\"token %s\" % token,\n",
    "        Accept = \"application/vnd.github.VERSION.sha\"\n",
    "    )\n",
    "    for repo in tqdm(repos):\n",
    "        request_sha = requests.get(\n",
    "            get_repo_sha_url(user, repo[\"name\"], repo[\"branch\"]),\n",
    "            headers=repo_sha_headers)\n",
    "        if request_sha.status_code == 409:\n",
    "            # Repo is empty\n",
    "            continue\n",
    "        else:\n",
    "            request_sha.raise_for_status()\n",
    "        repo[\"sha\"] = request_sha.text\n",
    "    return repos\n",
    "\n",
    "\n",
    "with open(repos_json, \"w\", encoding=\"utf8\") as fh:\n",
    "    json_dump(\n",
    "        list_repositories(\n",
    "            user=\"apache\",\n",
    "            # Generate a personal access token here\n",
    "            # https://github.com/settings/tokens\n",
    "            token=TOKEN,\n",
    "            max_size_mb=50,\n",
    "            repos_number=50),\n",
    "        fh\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from json import load as json_load\n",
    "from multiprocessing.pool import ThreadPool\n",
    "\n",
    "\n",
    "PARALLEL_DOWNLOADS = 10\n",
    "\n",
    "\n",
    "def clone_repo(name: str, clone_url: str, sha):\n",
    "    !cd {git_data_dir} \\\n",
    "        && git clone -q {clone_url} {name} \\\n",
    "        && cd {name} \\\n",
    "        && git checkout -q {sha}\n",
    "\n",
    "\n",
    "with ThreadPool(PARALLEL_DOWNLOADS) as pool, \\\n",
    "        open(repos_json, encoding=\"utf8\") as fh:\n",
    "    repos = json_load(fh)\n",
    "    pool.starmap(clone_repo,\n",
    "                 [(repo[\"name\"],\n",
    "                   repo[\"clone_url\"],\n",
    "                   repo[\"sha\"])\n",
    "                  for repo in repos])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}