From dedf727671563762b884c68fe6353a0adc8c93bb Mon Sep 17 00:00:00 2001 From: jhao Date: Wed, 8 Jul 2020 17:31:04 +0800 Subject: [PATCH] [update] :heavy_check_mark: readthedocs --- .gitignore | 1 + README.md | 1 + docs/Makefile | 20 +++++++++ docs/conf.py | 62 +++++++++++++++++++++++++++ docs/index.rst | 100 ++++++++++++++++++++++++++++++++++++++++++++ docs/make.bat | 35 ++++++++++++++++ docs/user/intro.rst | 8 ++++ 7 files changed, 227 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 docs/user/intro.rst diff --git a/.gitignore b/.gitignore index f09264408..4dae2645e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea/ +docs/_build *.pyc *.log diff --git a/README.md b/README.md index f0b67a83f..bde3ade29 100644 --- a/README.md +++ b/README.md @@ -228,4 +228,5 @@ PROXY_FETCHER = [ ### AD 　　最后, 开源不易, 有时间的小伙伴可以点下[推广](https://tracking.gitads.io/?repo=proxy_pool)广告。 + 　　[![AD](https://images.gitads.io/proxy_pool)](https://tracking.gitads.io/?repo=proxy_pool) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..d4bb2cbb9 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 000000000..8dd3b6d8f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,62 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'ProxyPool' +copyright = '2020, jhao104' +author = 'jhao104' + +# The full version, including alpha/beta/rc tags +release = '2.1.0' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'zh_CN' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 000000000..aa908471b --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,100 @@ +.. ProxyPool documentation master file, created by + sphinx-quickstart on Wed Jul 8 16:13:42 2020. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +ProxyPool +===================================== + +Python爬虫代理IP池 + +安装 +----- + +* 下载代码 + +.. code-block:: console + + $ git clone git@github.com:jhao104/proxy_pool.git + +* 安装依赖 + +.. code-block:: console + + $ pip install -r requirements.txt + +* 更新配置 + +.. code-block:: python + + HOST = "0.0.0.0" + PORT = 5000 + + DB_CONN = 'redis://@127.0.0.1:8888' + + PROXY_FETCHER = [ + "freeProxy01", + "freeProxy02", + # .... + ] + +* 启动项目 + +.. code-block:: console + + $ python proxyPool.py schedule + $ python proxyPool.py server + +使用 +______ + +* API + +============ ======== ================ ============== +Api Method Description Arg +============ ======== ================ ============== +/ GET API介绍 无 +/get GET 随机返回一个代理 无 +/get_all GET 返回所有代理 无 +/get_status GET 返回代理数量 无 +/delete GET 删除指定代理 proxy=host:ip +============ ======== ================ ============== + + +* 爬虫 + +.. code-block:: python + + import requests + + def get_proxy(): + return requests.get("http://127.0.0.1:5010/get/").json() + + def delete_proxy(proxy): + requests.get("http://127.0.0.1:5010/delete/?proxy={}".format(proxy)) + + # your spider code + + def getHtml(): + # .... + retry_count = 5 + proxy = get_proxy().get("proxy") + while retry_count > 0: + try: + html = requests.get('http://www.example.com', proxies={"http": "http://{}".format(proxy)}) + # 使用代理访问 + return html + except Exception: + retry_count -= 1 + # 删除代理池中代理 + delete_proxy(proxy) + return None + +Contents +-------- + +.. toctree:: + :maxdepth: 2 + + user/intro.rst + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 000000000..922152e96 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/user/intro.rst b/docs/user/intro.rst new file mode 100644 index 000000000..488a0c5d9 --- /dev/null +++ b/docs/user/intro.rst @@ -0,0 +1,8 @@ +.. intro + +用户手册 +======== + +安装 +------ +