diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e1749ee..26fc71b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - pythonversion: ["3.6", "3.7", "3.8", "3.9"] + pythonversion: ["3.7", "3.8", "3.9"] steps: - name: Checkout Repository uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index cf5782d..83d2696 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,28 +1,28 @@ # CHANGELOG -## NEXT RELEASE +## v4.0.0 (TODO) ### Breaking Changes -* The `--user_clone` and `--user_pull` flags are now titled `--personal_clone` and `--personal_pull` as the new `--user_clone` and `--user_pull` flags are used for a list of specified users -* Removes the `GITHUB_ARCHIVE_BUFFER` environment variable in favor of the new `--threads` flag -* Reworks the entire app config to use CLI flags solely instead of a mix of CLI flags and env variables, see the README for new usage instructions or run `github-archive --help` (closes #30) +* Reworks the entire app config to use CLI flags solely instead of a mix of CLI flags and env variables, additionally, most flags have changed names and functionality. See the README for new usage instructions or run `github-archive --help` (closes #30) * Repos or gists that fail to clone or pull will now be completely removed so that they can be retried from scratch on the next run of the tool. This was an especially important change for bad clones as the tool would previously leave an empty initialized git folder even if the clone failed which would not possess the actual git repo yet. In this state, you could not properly pull new changes because the content of the repo hadn't properly been cloned yet +* Bumps required Python version from 3.6 to 3.7 ### Features -* Adds the ability to specify a list of users via `GITHUB_ARCHIVE_USERS` to clone and pull repos for via the `--users_clone` and `--users_pull` flags (closes #20) -* Added the `--threads` flag which can specify the number of concurrent threads to run at once (closes #22) +* Adds a new `--users` flag which can be used to clone or pull git assets for a list of comma separated users (closes #20) +* Adds a new `--threads` flag which can specify the number of concurrent threads to run at once, default is `10` (closes #22) * Adds a new `--view` flag which allows you to "dry run" the application, seeing the entire list of repos and gists based on the input provided (closes #25) * Adds a new `--stars` flag which you can pass a comma separated list of users to and GitHub Archive will retrieve all of their starred repos which you can then view, clone, or pull (closes #26) -* Adds a new `--forks` flag which will include forks for whatever lists and operations you provide (closes #17)) +* Adds a new `--forks` flag which will include forks for whatever lists and operations you provide, default is `False` (closes #17)) ### Fixes * Removed verbose logging of skipped actions and "Already up to date" messages. Added additional logging related to API calls -* Adds proper validation of the `GITHUB_ARCHIVE_ORGS` variable on startup -* Various code refactor, bug fixes, and optimizations -* Bumped the default git operation timeout from 180 seconds to 300 seconds for larger repos (closes #22) +* Added proper validation and type checking of variables and environment on startup +* Various code refactors, bug fixes, and optimizations +* Bumped the default git operation timeout from `180 seconds` to `300 seconds` to assist with cloning or pulling larger repos (closes #22) +* Removes `mock` library in favor of builtin `unittest.mock` library ## v3.1.1 (2021-07-24) diff --git a/README.md b/README.md index e64eb05..6983d9f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # GitHub Archive -A powerful script to concurrently clone your entire GitHub instance or save it as an archive. +A powerful tool to concurrently clone or pull user and org repos and gists to create a GitHub archive. [![Build Status](https://github.com/Justintime50/github-archive/workflows/build/badge.svg)](https://github.com/Justintime50/github-archive/actions) [![Coverage Status](https://coveralls.io/repos/github/Justintime50/github-archive/badge.svg?branch=main)](https://coveralls.io/github/Justintime50/github-archive?branch=main) @@ -13,25 +13,11 @@ A powerful script to concurrently clone your entire GitHub instance or save it a -GitHub Archive will clone any repo and gist that doesn't exist locally and pull those that do from the main branch of each repo and latest revision of each gist that you have access to - including organizations (if configured). - -## What Can it Do? - -* Clone/pull personal repos (public and private) -* Clone/pull organization repos (public and private) -* Clone/pull personal gists (public and private) -* Iterate over infinite number of repos and gists concurrently -* Great use case: Run on a schedule to automate pulling changes or keep a local backup of all your repos +GitHub Archive is a powerful tool to concurrently clone or pull repositories or gists from GitHub with incredible flexibility. It's the perfect tool for spinning up a new dev environment, keeping a local copy of your GitHub instance, or quickly pulling in projects from your favorite users and organizations. ### Configurable Settings -The power of GitHub Archive comes in its configuration. Maybe you only want to clone/pull your personal public repos or maybe you want to go all out and include private repos from you and all organizations you belong to including your gists. Iterate over all your repos concurrently and sit back while GitHub Archive does the work. - -* Personal repos cloning/pulling -* Organization repos cloning/pulling -* Gists cloning/pulling -* List of organizations to include -* A host of environment variables to tweak GitHub Archive even further to meet your needs +The power of GitHub Archive comes in its configuration. Maybe you only want to clone or pull your personal public repos or maybe you want to go all out and include private repos from you and all organizations you belong to along with your gists. ## Install @@ -46,6 +32,35 @@ make install make help ``` +## Usage + +``` +Usage: + github-archive --users justintime50 --clone + +Options: + -h, --help show this help message and exit + -v, --view Pass this flag to view git assets (dry run). + -c, --clone Pass this flag to clone git assets. + -p, --pull Pass this flag to pull git assets. + -f, --forks Pass this flag to include forked git assets. + -u USERS, --users USERS + Pass a comma separated list of users to get repos for. + -o ORGS, --orgs ORGS Pass a comma separated list of orgs to get repos for. + -g GISTS, --gists GISTS + Pass a comma separated list of users to get gists for. + -s STARS, --stars STARS + Pass a comma separated list of users to get starred repos for. + -to TIMEOUT, --timeout TIMEOUT + The number of seconds before a git operation times out. + -th THREADS, --threads THREADS + The number of concurrent threads to run. + -t TOKEN, --token TOKEN + Provide your GitHub token to authenticate with the GitHub API and gain access to private repos and gists. + -l LOCATION, --location LOCATION + The location where you want your GitHub Archive to be stored. +``` + ### Automating SSH Passphrase Prompt (Recommended) To allow the script to run continuosly without requiring your SSH passphrase, you'll need to add your passphrase to the SSH agent. **NOTE:** Your SSH passphrase will be unloaded upon logout. @@ -55,38 +70,13 @@ To allow the script to run continuosly without requiring your SSH passphrase, yo ssh-add ``` -## Usage +### Notes **SSH Key:** You must have an SSH key generated on your local machine and added to your GitHub account as this tool uses the `ssh_url` to clone/pull. **Merge Conflicts:** Be aware that using GitHub Archive could lead to merge conflicts if you do not commit or stash your changes if using these repos as active development repos instead of simply an archive or one-time clone. -``` -Basic Usage: - GITHUB_TOKEN=123... github-archive --user-clone --user-pull - -Advanced Usage: - GITHUB_TOKEN=123... GITHUB_ARCHIVE_ORGS="org1, org2" GITHUB_ARCHIVE_LOCATION="~/custom_location" \ - github-archive -uc -up -gc -gp -oc -op - -Options: - -h, --help show this help message and exit - -uc, --user_clone Clone personal repos. - -up, --user_pull Pull personal repos - -gc, --gists_clone Clone personal gists - -gp, --gists_pull Pull personal gists. - -oc, --orgs_clone Clone organization repos. - -op, --orgs_pull Pull organization repos. - -Environment Variables: - GITHUB_TOKEN expects a string of your GitHub Token - GITHUB_ARCHIVE_ORGS expects a string of comma separated orgs. eg: "org1, org2" - GITHUB_ARCHIVE_LOCATION expects a string of an explicit location on your machine (eg: "~/custom_location"). Default: ~/github-archive - GITHUB_ARCHIVE_BUFFER expects a float for the buffer inbetween requests. Default: 0.1 - GITHUB_ARCHIVE_TIMEOUT expects an int for the number of seconds before a git operation times out. Default: 180 - GITHUB_ARCHIVE_LOG_MAX_BYTES expects an int of the max bytes that a log will grow to. Once the log exceeds this number, it will rollover to another log. Default: 200000 - GITHUB_ARCHIVE_LOG_BACKUP_COUNT expects an int of the number of logs to rollover once a single log exceeds the max bytes size. Default: 5 -``` +**Access**: GitHub Archive can only clone or pull repos that the authenticated user has access to. This means that private repos from another user or org that you don't have access to will not be able to be cloned or pulled. ## Development @@ -103,7 +93,3 @@ make coverage # Run the tool locally venv/bin/python github_archive/cli.py --help ``` - -## Legacy Script - -This tool was initially built in Bash and later re-written in Python. If you'd like to use or view the legacy script, check out the separate [Legacy README](legacy/README.md). diff --git a/README_v4.md b/README_v4.md deleted file mode 100644 index 29b3bda..0000000 --- a/README_v4.md +++ /dev/null @@ -1,107 +0,0 @@ -
- -# GitHub Archive - -A powerful tool to concurrently clone or pull user and org repos and gists to create a GitHub archive. - -[![Build Status](https://github.com/Justintime50/github-archive/workflows/build/badge.svg)](https://github.com/Justintime50/github-archive/actions) -[![Coverage Status](https://coveralls.io/repos/github/Justintime50/github-archive/badge.svg?branch=main)](https://coveralls.io/github/Justintime50/github-archive?branch=main) -[![PyPi](https://img.shields.io/pypi/v/github-archive)](https://pypi.org/project/github-archive) -[![Licence](https://img.shields.io/github/license/justintime50/GitHub-archive)](LICENSE) - -Showcase - -
- -GitHub Archive will clone any repo and gist that doesn't exist locally and pull those that do from the main branch of each repo and latest revision of each gist that you have access to - including organizations (if configured). - -## What Can it Do? - -* Clone/pull personal repos (public and private) -* Clone/pull organization repos (public and private) -* Clone/pull personal gists (public and private) -* Iterate over infinite number of repos and gists concurrently -* Great use case: Run on a schedule to automate pulling changes or keep a local backup of all your repos - -### Configurable Settings - -The power of GitHub Archive comes in its configuration. Maybe you only want to clone/pull your personal public repos or maybe you want to go all out and include private repos from you and all organizations you belong to including your gists. Iterate over all your repos concurrently and sit back while GitHub Archive does the work. - -* Personal repos cloning/pulling -* Organization repos cloning/pulling -* Gists cloning/pulling -* List of organizations to include -* A host of environment variables to tweak GitHub Archive even further to meet your needs - -## Install - -```bash -# Install tool -pip3 install github-archive - -# Install locally -make install - -# Get Makefile help -make help -``` - -### Automating SSH Passphrase Prompt (Recommended) - -To allow the script to run continuosly without requiring your SSH passphrase, you'll need to add your passphrase to the SSH agent. **NOTE:** Your SSH passphrase will be unloaded upon logout. - -```bash -# This assumes you've saved your SSH keys to the default location -ssh-add -``` - -## Usage - -**SSH Key:** You must have an SSH key generated on your local machine and added to your GitHub account as this tool uses the `ssh_url` to clone/pull. - -**Merge Conflicts:** Be aware that using GitHub Archive could lead to merge conflicts if you do not commit or stash your changes if using these repos as active development repos instead of simply an archive or one-time clone. - -**Access**: GitHub Archive can only clone or pull repos that the authenticated user has access to. That means that private repos from another user or orgs that you don't have access to will not be able to be cloned or pulled. - -``` -Usage: - github-archive --users justintime50 --clone - -Options: - -h, --help show this help message and exit - -v, --view Pass this flag to view git assets (dry run). - -c, --clone Pass this flag to clone git assets. - -p, --pull Pass this flag to pull git assets. - -f, --forks Pass this flag to include forked git assets. - -u USERS, --users USERS - Pass a comma separated list of users to get repos for. - -o ORGS, --orgs ORGS Pass a comma separated list of orgs to get repos for. - -g GISTS, --gists GISTS - Pass a comma separated list of users to get gists for. - -s STARS, --stars STARS - Pass a comma separated list of users to get starred repos for. - -to TIMEOUT, --timeout TIMEOUT - The number of seconds before a git operation times out. - -th THREADS, --threads THREADS - The number of concurrent threads to run. - -t TOKEN, --token TOKEN - Provide your GitHub token to authenticate with the GitHub API and gain access to private repos and gists. - -l LOCATION, --location LOCATION - The location where you want your GitHub Archive to be stored. -``` - -## Development - -```bash -# Lint the project -make lint - -# Run tests -make test - -# Run test coverage -make coverage - -# Run the tool locally -venv/bin/python github_archive/cli.py --help -``` diff --git a/setup.py b/setup.py index 6a5e14d..d2c8e21 100644 --- a/setup.py +++ b/setup.py @@ -10,15 +10,16 @@ DEV_REQUIREMENTS = [ 'coveralls == 3.*', 'flake8', - 'mock == 4.*', 'pytest == 6.*', 'pytest-cov == 2.*', ] setuptools.setup( name='github-archive', - version='3.1.1', - description='A powerful tool to concurrently clone or pull user and org repos and gists to create a GitHub archive.', # noqa + version='4.0.0', + description=( + 'A powerful tool to concurrently clone or pull user and org repos and gists to create a GitHub archive.' + ), long_description=long_description, long_description_content_type="text/markdown", url='http://github.com/justintime50/github-archive', @@ -39,5 +40,5 @@ 'github-archive=github_archive.cli:main', ] }, - python_requires='>=3.6', + python_requires='>=3.7', ) diff --git a/test/unit/conftest.py b/test/unit/conftest.py index 232ab54..bdb29b1 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -1,4 +1,5 @@ -import mock +from unittest.mock import MagicMock + import pytest @@ -7,7 +8,7 @@ def mock_git_asset(): """This can be used for repos and/or gists, it contains shared data for either git asset for easier testing. """ - mock_git_asset = mock.MagicMock() + mock_git_asset = MagicMock() mock_git_asset.id = '123' mock_git_asset.name = 'mock-asset-name' mock_git_asset.owner.name = 'Mock User Name' diff --git a/test/unit/test_archive.py b/test/unit/test_archive.py index f157af0..056e0ec 100644 --- a/test/unit/test_archive.py +++ b/test/unit/test_archive.py @@ -1,7 +1,7 @@ import subprocess from threading import BoundedSemaphore +from unittest.mock import patch -import mock import pytest from github_archive import GithubArchive from github_archive.archive import ( @@ -14,10 +14,6 @@ USER_CONTEXT, ) -GITHUB_TOKEN = '123' -ORG_LIST = 'org1, org2' -USER_LIST = 'user1, user2' - def mock_thread_limiter(): thread_limiter = BoundedSemaphore(DEFAULT_NUM_THREADS) @@ -25,10 +21,10 @@ def mock_thread_limiter(): return thread_limiter -@mock.patch('github_archive.archive.Github.get_user') -@mock.patch('github_archive.archive.GithubArchive.authenticated_user_in_users', return_value=True) -@mock.patch('github_archive.archive.GithubArchive.view_repos') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.Github.get_user') +@patch('github_archive.archive.GithubArchive.authenticated_user_in_users', return_value=True) +@patch('github_archive.archive.GithubArchive.view_repos') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_token_view(mock_get_all_git_assets, mock_view_repos, mock_authed_user_in_users, mock_get_user): github_archive = GithubArchive( token='123', @@ -44,10 +40,10 @@ def test_run_token_view(mock_get_all_git_assets, mock_view_repos, mock_authed_us assert github_archive.users == [] # Assert the authed user gets removed from list -@mock.patch('github_archive.archive.Github.get_user') -@mock.patch('github_archive.archive.GithubArchive.authenticated_user_in_users', return_value=True) -@mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.Github.get_user') +@patch('github_archive.archive.GithubArchive.authenticated_user_in_users', return_value=True) +@patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_token_clone( mock_get_all_git_assets, mock_iterate_repos_to_archive, mock_authed_user_in_users, mock_get_user ): @@ -65,10 +61,10 @@ def test_run_token_clone( assert github_archive.users == [] # Assert the authed user gets removed from list -@mock.patch('github_archive.archive.Github.get_user') -@mock.patch('github_archive.archive.GithubArchive.authenticated_user_in_users', return_value=True) -@mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.Github.get_user') +@patch('github_archive.archive.GithubArchive.authenticated_user_in_users', return_value=True) +@patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_token_pull( mock_get_all_git_assets, mock_iterate_repos_to_archive, mock_authed_user_in_users, mock_get_user ): @@ -86,8 +82,8 @@ def test_run_token_pull( assert github_archive.users == [] # Assert the authed user gets removed from list -@mock.patch('github_archive.archive.GithubArchive.view_repos') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.view_repos') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_users_view(mock_get_all_git_assets, mock_view_repos): GithubArchive( users='justintime50', @@ -98,8 +94,8 @@ def test_run_users_view(mock_get_all_git_assets, mock_view_repos): mock_view_repos.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_users_clone(mock_get_all_git_assets, mock_iterate_repos_to_archive): GithubArchive( users='justintime50', @@ -110,8 +106,8 @@ def test_run_users_clone(mock_get_all_git_assets, mock_iterate_repos_to_archive) mock_iterate_repos_to_archive.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_users_pull(mock_get_all_git_assets, mock_iterate_repos_to_archive): GithubArchive( users='justintime50', @@ -122,8 +118,8 @@ def test_run_users_pull(mock_get_all_git_assets, mock_iterate_repos_to_archive): mock_iterate_repos_to_archive.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.view_repos') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.view_repos') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_orgs_view(mock_get_all_git_assets, mock_view_repos): GithubArchive( orgs='org1', @@ -134,8 +130,8 @@ def test_run_orgs_view(mock_get_all_git_assets, mock_view_repos): mock_view_repos.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_orgs_clone(mock_get_all_git_assets, mock_iterate_repos_to_archive): GithubArchive( orgs='org1', @@ -146,8 +142,8 @@ def test_run_orgs_clone(mock_get_all_git_assets, mock_iterate_repos_to_archive): mock_iterate_repos_to_archive.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_orgs_pull(mock_get_all_git_assets, mock_iterate_repos_to_archive): GithubArchive( orgs='org1', @@ -158,8 +154,8 @@ def test_run_orgs_pull(mock_get_all_git_assets, mock_iterate_repos_to_archive): mock_iterate_repos_to_archive.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.view_gists') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.view_gists') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_gists_view(mock_get_all_git_assets, mock_view_gists): GithubArchive( gists='justintime50', @@ -170,8 +166,8 @@ def test_run_gists_view(mock_get_all_git_assets, mock_view_gists): mock_view_gists.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.iterate_gists_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.iterate_gists_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_gists_clone(mock_get_all_git_assets, mock_iterate_gists_to_archive): GithubArchive( gists='org1', @@ -182,8 +178,8 @@ def test_run_gists_clone(mock_get_all_git_assets, mock_iterate_gists_to_archive) mock_iterate_gists_to_archive.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.iterate_gists_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.iterate_gists_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_gists_pull(mock_get_all_git_assets, mock_iterate_gists_to_archive): GithubArchive( gists='org1', @@ -194,8 +190,8 @@ def test_run_gists_pull(mock_get_all_git_assets, mock_iterate_gists_to_archive): mock_iterate_gists_to_archive.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.view_repos') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.view_repos') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_stars_view(mock_get_all_git_assets, mock_view_repos): GithubArchive( stars='justintime50', @@ -206,8 +202,8 @@ def test_run_stars_view(mock_get_all_git_assets, mock_view_repos): mock_view_repos.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_stars_clone(mock_get_all_git_assets, mock_iterate_repos_to_archive): GithubArchive( stars='justintime50', @@ -218,8 +214,8 @@ def test_run_stars_clone(mock_get_all_git_assets, mock_iterate_repos_to_archive) mock_iterate_repos_to_archive.assert_called_once() -@mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') -@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets') +@patch('github_archive.archive.GithubArchive.iterate_repos_to_archive') +@patch('github_archive.archive.GithubArchive.get_all_git_assets') def test_run_stars_pull(mock_get_all_git_assets, mock_iterate_repos_to_archive): GithubArchive( stars='justintime50', @@ -230,15 +226,15 @@ def test_run_stars_pull(mock_get_all_git_assets, mock_iterate_repos_to_archive): mock_iterate_repos_to_archive.assert_called_once() -@mock.patch('os.path.exists', return_value=False) -@mock.patch('os.makedirs') +@patch('os.path.exists', return_value=False) +@patch('os.makedirs') def test_initialize_project(mock_make_dirs, mock_dir_exist): GithubArchive().initialize_project() assert mock_make_dirs.call_count == 2 -@mock.patch('github_archive.archive.LOGGER') +@patch('github_archive.archive.LOGGER') def test_initialize_project_missing_list(mock_logger): # TODO: Is it possible to test all variations easily in one test? # Parametrize doesn't work great because we can't easily swap the param name being used @@ -250,7 +246,7 @@ def test_initialize_project_missing_list(mock_logger): assert message == str(error.value) -@mock.patch('github_archive.archive.LOGGER') +@patch('github_archive.archive.LOGGER') def test_initialize_project_missing_operation(mock_logger): # TODO: Is it possible to test all variations easily in one test? # Parametrize doesn't work great because we can't easily swap the param name being used @@ -264,7 +260,7 @@ def test_initialize_project_missing_operation(mock_logger): assert message == str(error.value) -@mock.patch('github_archive.archive.Github.get_user') +@patch('github_archive.archive.Github.get_user') def test_authenticated_user_in_users(mock_get_user): authenticated_user_in_users = GithubArchive( token='123', @@ -274,7 +270,7 @@ def test_authenticated_user_in_users(mock_get_user): assert authenticated_user_in_users is False -@mock.patch('github_archive.archive.Github.get_user') +@patch('github_archive.archive.Github.get_user') def test_get_all_git_assets(mock_get_user): GithubArchive( users='justintime50', @@ -283,9 +279,9 @@ def test_get_all_git_assets(mock_get_user): mock_get_user.assert_called_once() -@mock.patch('github_archive.archive.Github') -@mock.patch('github_archive.archive.Github.get_repos') -@mock.patch('github_archive.archive.Github.get_user') +@patch('github_archive.archive.Github') +@patch('github_archive.archive.Github.get_repos') +@patch('github_archive.archive.Github.get_user') def test_get_all_user_repos(mock_get_user, mock_get_repos, mock_github_instance): users = 'justintime50,user2' GithubArchive( @@ -297,9 +293,9 @@ def test_get_all_user_repos(mock_get_user, mock_get_repos, mock_github_instance) # TODO: Assert the list returned -@mock.patch('github_archive.archive.Github') -@mock.patch('github_archive.archive.Github.get_repos') -@mock.patch('github_archive.archive.Github.get_organization') +@patch('github_archive.archive.Github') +@patch('github_archive.archive.Github.get_repos') +@patch('github_archive.archive.Github.get_organization') def test_get_all_org_repos(mock_get_org, mock_get_repos, mock_github_instance): orgs = 'org1,org2' GithubArchive( @@ -311,9 +307,9 @@ def test_get_all_org_repos(mock_get_org, mock_get_repos, mock_github_instance): # TODO: Assert the list returned -@mock.patch('github_archive.archive.Github') -@mock.patch('github_archive.archive.Github.get_gists') -@mock.patch('github_archive.archive.Github.get_user') +@patch('github_archive.archive.Github') +@patch('github_archive.archive.Github.get_gists') +@patch('github_archive.archive.Github.get_user') def test_get_get_all_gists(mock_get_user, mock_get_gists, mock_github_instance): gists = 'justintime50,user2' GithubArchive( @@ -325,8 +321,8 @@ def test_get_get_all_gists(mock_get_user, mock_get_gists, mock_github_instance): # TODO: Assert the list returned -@mock.patch('github_archive.archive.Github') -@mock.patch('github_archive.archive.GithubArchive.archive_repo') +@patch('github_archive.archive.Github') +@patch('github_archive.archive.GithubArchive.archive_repo') def test_iterate_repos_not_matching_authed_username(mock_archive_repo, mock_github_instance, mock_git_asset): repos = [mock_git_asset] GithubArchive( @@ -336,8 +332,8 @@ def test_iterate_repos_not_matching_authed_username(mock_archive_repo, mock_gith mock_archive_repo.assert_called_once() -@mock.patch('github_archive.archive.Github') -@mock.patch('github_archive.archive.GithubArchive.archive_repo') +@patch('github_archive.archive.Github') +@patch('github_archive.archive.GithubArchive.archive_repo') def test_iterate_repos_matching_authed_username(mock_archive_repo, mock_github_instance, mock_git_asset): repos = [mock_git_asset] GithubArchive( @@ -348,8 +344,8 @@ def test_iterate_repos_matching_authed_username(mock_archive_repo, mock_github_i mock_archive_repo.assert_called_once() -@mock.patch('github_archive.archive.Github') -@mock.patch('github_archive.archive.GithubArchive.archive_gist') +@patch('github_archive.archive.Github') +@patch('github_archive.archive.GithubArchive.archive_gist') def test_iterate_gists(mock_archive_gist, mock_github_instance, mock_git_asset): gists = [mock_git_asset] GithubArchive( @@ -359,7 +355,7 @@ def test_iterate_gists(mock_archive_gist, mock_github_instance, mock_git_asset): mock_archive_gist.assert_called() -@mock.patch('github_archive.archive.LOGGER') +@patch('github_archive.archive.LOGGER') def test_view_repos(mock_logger, mock_git_asset): repos = [mock_git_asset] GithubArchive().view_repos(repos) @@ -367,7 +363,7 @@ def test_view_repos(mock_logger, mock_git_asset): mock_logger.info.assert_called_with('mock_username/mock-asset-name') -@mock.patch('github_archive.archive.LOGGER') +@patch('github_archive.archive.LOGGER') def test_view_gists(mock_logger, mock_git_asset): gists = [mock_git_asset] GithubArchive().view_gists(gists) @@ -375,8 +371,8 @@ def test_view_gists(mock_logger, mock_git_asset): mock_logger.info.assert_called_with('mock_username/123') -@mock.patch('subprocess.run') -@mock.patch('github_archive.archive.LOGGER') +@patch('subprocess.run') +@patch('github_archive.archive.LOGGER') def test_archive_repo_success(mock_logger, mock_subprocess, mock_git_asset): # TODO: Mock the subprocess better to ensure it's doing what it should operation = CLONE_OPERATION @@ -387,9 +383,9 @@ def test_archive_repo_success(mock_logger, mock_subprocess, mock_git_asset): mock_logger.info.assert_called_once_with(message) -@mock.patch('os.path.exists', return_value=True) -@mock.patch('subprocess.run') -@mock.patch('github_archive.archive.LOGGER') +@patch('os.path.exists', return_value=True) +@patch('subprocess.run') +@patch('github_archive.archive.LOGGER') def test_archive_repo_clone_exists(mock_logger, mock_subprocess, mock_git_asset): operation = CLONE_OPERATION GithubArchive().archive_repo(mock_thread_limiter(), mock_git_asset, 'assets', operation) @@ -397,9 +393,9 @@ def test_archive_repo_clone_exists(mock_logger, mock_subprocess, mock_git_asset) mock_subprocess.assert_not_called() -@mock.patch('shutil.rmtree') -@mock.patch('subprocess.run', side_effect=subprocess.TimeoutExpired(cmd=subprocess.run, timeout=0.1)) -@mock.patch('github_archive.archive.LOGGER') +@patch('shutil.rmtree') +@patch('subprocess.run', side_effect=subprocess.TimeoutExpired(cmd=subprocess.run, timeout=0.1)) +@patch('github_archive.archive.LOGGER') def test_archive_repo_timeout_exception(mock_logger, mock_subprocess, mock_remove_dir, mock_git_asset): operation = CLONE_OPERATION message = f'Git operation timed out archiving {mock_git_asset.name}.' @@ -410,9 +406,9 @@ def test_archive_repo_timeout_exception(mock_logger, mock_subprocess, mock_remov # mock_remove_dir.assert_called_once_with('mock/path') -@mock.patch('shutil.rmtree') -@mock.patch('subprocess.run', side_effect=subprocess.CalledProcessError(returncode=1, cmd=subprocess.run)) -@mock.patch('github_archive.archive.LOGGER') +@patch('shutil.rmtree') +@patch('subprocess.run', side_effect=subprocess.CalledProcessError(returncode=1, cmd=subprocess.run)) +@patch('github_archive.archive.LOGGER') def test_archive_repo_called_process_error(mock_logger, mock_subprocess, mock_remove_dir, mock_git_asset): operation = PULL_OPERATION GithubArchive().archive_repo(mock_thread_limiter(), mock_git_asset, 'assets', operation) @@ -422,8 +418,8 @@ def test_archive_repo_called_process_error(mock_logger, mock_subprocess, mock_re # mock_remove_dir.assert_called_once_with('mock/path') -@mock.patch('subprocess.run') -@mock.patch('github_archive.archive.LOGGER') +@patch('subprocess.run') +@patch('github_archive.archive.LOGGER') def test_archive_gist_success(mock_logger, mock_subprocess, mock_git_asset): # TODO: Mock the subprocess better to ensure it's doing what it should operation = CLONE_OPERATION @@ -434,9 +430,9 @@ def test_archive_gist_success(mock_logger, mock_subprocess, mock_git_asset): mock_logger.info.assert_called_once_with(message) -@mock.patch('os.path.exists', return_value=True) -@mock.patch('subprocess.run') -@mock.patch('github_archive.archive.LOGGER') +@patch('os.path.exists', return_value=True) +@patch('subprocess.run') +@patch('github_archive.archive.LOGGER') def test_archive_gist_clone_exists(mock_logger, mock_subprocess, mock_path_exists, mock_git_asset): operation = CLONE_OPERATION GithubArchive().archive_gist(mock_thread_limiter(), mock_git_asset, 'assets', operation) @@ -444,9 +440,9 @@ def test_archive_gist_clone_exists(mock_logger, mock_subprocess, mock_path_exist mock_subprocess.assert_not_called() -@mock.patch('shutil.rmtree') -@mock.patch('subprocess.run', side_effect=subprocess.TimeoutExpired(cmd=subprocess.run, timeout=0.1)) -@mock.patch('github_archive.archive.LOGGER') +@patch('shutil.rmtree') +@patch('subprocess.run', side_effect=subprocess.TimeoutExpired(cmd=subprocess.run, timeout=0.1)) +@patch('github_archive.archive.LOGGER') def test_archive_gist_timeout_exception(mock_logger, mock_subprocess, mock_remove_dir, mock_git_asset): operation = CLONE_OPERATION message = f'Git operation timed out archiving {mock_git_asset.id}.' @@ -457,9 +453,9 @@ def test_archive_gist_timeout_exception(mock_logger, mock_subprocess, mock_remov # mock_remove_dir.assert_called_once_with('mock/path') -@mock.patch('shutil.rmtree') -@mock.patch('subprocess.run', side_effect=subprocess.CalledProcessError(returncode=1, cmd=subprocess.run)) -@mock.patch('github_archive.archive.LOGGER') +@patch('shutil.rmtree') +@patch('subprocess.run', side_effect=subprocess.CalledProcessError(returncode=1, cmd=subprocess.run)) +@patch('github_archive.archive.LOGGER') def test_archive_gist_called_process_error(mock_logger, mock_subprocess, mock_remove_dir, mock_git_asset): operation = PULL_OPERATION GithubArchive().archive_gist(mock_thread_limiter(), mock_git_asset, 'assets', operation) diff --git a/test/unit/test_logger.py b/test/unit/test_logger.py index b2850b4..12bee13 100644 --- a/test/unit/test_logger.py +++ b/test/unit/test_logger.py @@ -1,30 +1,30 @@ import os +from unittest.mock import mock_open, patch -import mock from github_archive.logger import Logger LOG_PATH = 'test/mock-dir' LOG_FILE = './test/test.log' -@mock.patch('os.makedirs') -@mock.patch('github_archive.archive.LOGGER') +@patch('os.makedirs') +@patch('github_archive.archive.LOGGER') def test_setup_logging(mock_logger, mock_make_dirs): - with mock.patch('builtins.open', mock.mock_open()): + with patch('builtins.open', mock_open()): Logger.setup_logging(mock_logger, LOG_PATH) mock_make_dirs.assert_called_once() mock_logger.setLevel.assert_called() mock_logger.addHandler.assert_called() -@mock.patch('os.makedirs') -@mock.patch('github_archive.archive.LOGGER') +@patch('os.makedirs') +@patch('github_archive.archive.LOGGER') def test_setup_logging_dir_exists(mock_logger, mock_make_dirs): # TODO: Mock this better so we don't need a gitignored empty folder for testing if not os.path.exists('./logs'): os.mkdir('logs') - with mock.patch('builtins.open', mock.mock_open()): + with patch('builtins.open', mock_open()): Logger.setup_logging(mock_logger, './') mock_make_dirs.assert_not_called() mock_logger.setLevel.assert_called()