From 3ccabd25309041d422cae97f5aecbf93303efd88 Mon Sep 17 00:00:00 2001
From: jhao <jingh@jccfc.com>
Date: Tue, 3 Nov 2020 17:10:12 +0800
Subject: [PATCH 1/6] [update] show proxy count

---
 test/testProxyFetcher.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/test/testProxyFetcher.py b/test/testProxyFetcher.py
index 7a782f705..a530b2e50 100644
--- a/test/testProxyFetcher.py
+++ b/test/testProxyFetcher.py
@@ -19,13 +19,14 @@
 def testProxyFetcher():
     conf = ConfigHandler()
     proxy_getter_functions = conf.fetchers
+    proxy_counter = {_: 0 for _ in proxy_getter_functions}
     for proxyGetter in proxy_getter_functions:
-        proxy_count = 0
         for proxy in getattr(ProxyFetcher, proxyGetter.strip())():
             if proxy:
-                print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy,
-                                                                                     proxy_count=proxy_count))
-                proxy_count += 1
+                print('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
+                proxy_counter[proxyGetter] = proxy_counter.get(proxyGetter) + 1
+    for key, value in proxy_counter.items():
+        print(key, value)
 
 
 if __name__ == '__main__':

From 69e4f862fc0a847404fcef6e39ab13da9edd2b3b Mon Sep 17 00:00:00 2001
From: jhao <jingh@jccfc.com>
Date: Wed, 4 Nov 2020 10:35:15 +0800
Subject: [PATCH 2/6] =?UTF-8?q?[update]=20=E4=BB=A3=E7=90=8666?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 fetcher/proxyFetcher.py | 47 ++++++-----------------------------------
 1 file changed, 6 insertions(+), 41 deletions(-)

diff --git a/fetcher/proxyFetcher.py b/fetcher/proxyFetcher.py
index 71cf276b0..233cacbd1 100644
--- a/fetcher/proxyFetcher.py
+++ b/fetcher/proxyFetcher.py
@@ -54,52 +54,17 @@ def freeProxy01():
                     print(e)
 
     @staticmethod
-    def freeProxy02(count=20):
+    def freeProxy02():
         """
         代理66 http://www.66ip.cn/
-        :param count: 提取数量
         :return:
         """
-        urls = [
-            "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=",
-            "http://www.66ip.cn/nmtq.php?getnum={}&isp=0&anonymoustype=0&s"
-            "tart=&ports=&export=&ipaddress=&area=0&proxytype=2&api=66ip"
-        ]
-
-        try:
-            import execjs
-            import requests
-
-            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
-                       'Accept': '*/*',
-                       'Connection': 'keep-alive',
-                       'Accept-Language': 'zh-CN,zh;q=0.8'}
-            session = requests.session()
-            src = session.get("http://www.66ip.cn/", headers=headers).text
-            src = src.split("</script>")[0] + '}'
-            src = src.replace("<script>", "function test() {")
-            src = src.replace("while(z++)try{eval(", ';var num=10;while(z++)try{var tmp=')
-            src = src.replace(");break}", ";num--;if(tmp.search('cookie') != -1 | num<0){return tmp}}")
-            ctx = execjs.compile(src)
-            src = ctx.call("test")
-            src = src[src.find("document.cookie="): src.find("};if((")]
-            src = src.replace("document.cookie=", "")
-            src = "function test() {var window={}; return %s }" % src
-            cookie = execjs.compile(src).call('test')
-            js_cookie = cookie.split(";")[0].split("=")[-1]
-        except Exception as e:
-            print(e)
-            return
+        url = "http://www.66ip.cn/mo.php"
 
-        for url in urls:
-            try:
-                html = session.get(url.format(count), cookies={"__jsl_clearance": js_cookie}, headers=headers).text
-                ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", html)
-                for ip in ips:
-                    yield ip.strip()
-            except Exception as e:
-                print(e)
-                pass
+        resp = WebRequest().get(url, timeout=10)
+        proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5})', resp.text)
+        for proxy in proxies:
+            yield proxy
 
     @staticmethod
     def freeProxy03(page_count=1):

From 68663ba86a7982362dbc0183b3ce46f72c386c4b Mon Sep 17 00:00:00 2001
From: jhao <jingh@jccfc.com>
Date: Wed, 4 Nov 2020 11:39:45 +0800
Subject: [PATCH 3/6] =?UTF-8?q?[update]=20=E5=85=A8=E7=BD=91=E4=BB=A3?=
 =?UTF-8?q?=E7=90=86=20=E7=AB=AF=E5=8F=A3=E5=8A=A0=E5=AF=86?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 fetcher/proxyFetcher.py | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/fetcher/proxyFetcher.py b/fetcher/proxyFetcher.py
index 233cacbd1..f5442d499 100644
--- a/fetcher/proxyFetcher.py
+++ b/fetcher/proxyFetcher.py
@@ -69,7 +69,7 @@ def freeProxy02():
     @staticmethod
     def freeProxy03(page_count=1):
         """
-        西刺代理 http://www.xicidaili.com
+        西刺代理 http://www.xicidaili.com  网站已关闭
         :return:
         """
         url_list = [
@@ -90,7 +90,7 @@ def freeProxy03(page_count=1):
     @staticmethod
     def freeProxy04():
         """
-        guobanjia http://www.goubanjia.com/
+        全网代理 http://www.goubanjia.com/
         :return:
         """
         url = "http://www.goubanjia.com/"
@@ -103,25 +103,22 @@ def freeProxy04():
                                         and not(contains(@class, 'port'))
                                         ]/text()
                                 """
+
+        # port是class属性值加密得到
+        def _parse_port(port_element):
+            port_list = []
+            for letter in port_element:
+                port_list.append(str("ABCDEFGHIZ".find(letter)))
+            _port = "".join(port_list)
+            return int(_port) >> 0x3
+
         for each_proxy in proxy_list:
             try:
-                # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port
                 ip_addr = ''.join(each_proxy.xpath(xpath_str))
-
-                # HTML中的port是随机数，真正的端口编码在class后面的字母中。
-                # 比如这个：
-                # <span class="port CFACE">9054</span>
-                # CFACE解码后对应的是3128。
-                port = 0
-                for _ in each_proxy.xpath(".//span[contains(@class, 'port')]"
-                                          "/attribute::class")[0]. \
-                        replace("port ", ""):
-                    port *= 10
-                    port += (ord(_) - ord('A'))
-                port /= 8
-
+                port_str = each_proxy.xpath(".//span[contains(@class, 'port')]/@class")[0].split()[-1]
+                port = _parse_port(port_str.strip())
                 yield '{}:{}'.format(ip_addr, int(port))
-            except Exception as e:
+            except Exception:
                 pass
 
     @staticmethod

From 38002d8270bcc4d0e672670b6e661ccef008cc06 Mon Sep 17 00:00:00 2001
From: jhao <jingh@jccfc.com>
Date: Wed, 4 Nov 2020 13:36:29 +0800
Subject: [PATCH 4/6] =?UTF-8?q?[update]=20=E4=BB=A3=E7=90=86=E7=9B=92?=
 =?UTF-8?q?=E5=AD=90?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 fetcher/proxyFetcher.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/fetcher/proxyFetcher.py b/fetcher/proxyFetcher.py
index f5442d499..d0b42996f 100644
--- a/fetcher/proxyFetcher.py
+++ b/fetcher/proxyFetcher.py
@@ -145,15 +145,18 @@ def freeProxy05(page_count=1):
     @staticmethod
     def freeProxy06():
         """
-        码农代理 https://proxy.coderbusy.com/
+        代理盒子 https://proxy.coderbusy.com/
         :return:
         """
-        urls = ['https://proxy.coderbusy.com/']
+        urls = ['https://proxy.coderbusy.com/zh-hans/ops/country/cn.html']
         for url in urls:
             tree = WebRequest().get(url).tree
             proxy_list = tree.xpath('.//table//tr')
             for tr in proxy_list[1:]:
-                yield ':'.join(tr.xpath('./td/text()')[0:2])
+                proxy = '{}:{}'.format("".join(tr.xpath("./td[1]/text()")).strip(),
+                                       "".join(tr.xpath("./td[2]//text()")).strip())
+                if proxy:
+                    yield proxy
 
     @staticmethod
     def freeProxy07():

From ecc5c94eb3205efb3d74eb6c3c9942be6f5d04ea Mon Sep 17 00:00:00 2001
From: jhao <jingh@jccfc.com>
Date: Wed, 4 Nov 2020 14:50:00 +0800
Subject: [PATCH 5/6] [update] proxy

---
 README.md               | 28 ++++++++++++++--------------
 docs/changelog.rst      |  3 +++
 fetcher/proxyFetcher.py | 25 ++++++-------------------
 setting.py              |  7 +++----
 4 files changed, 26 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index 3a5dfabc5..56884add9 100644
--- a/README.md
+++ b/README.md
@@ -190,20 +190,20 @@ PROXY_FETCHER = [
 
    目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): 
    
-  | 厂商名称 |  状态  |  更新速度 |  可用率  |  是否被墙  |  地址 |
-  | -----   |  ---- | --------  | ------ | --------- | ----- |
-  | 无忧代理 |  可用  | 几分钟一次 |   *     |  否       | [地址](http://www.data5u.com/free/index.html) |
-  | 66代理   | 可用  | 更新很慢   |   *     |  否      | [地址](http://www.66ip.cn/) |
-  | 西刺代理 | 可用   | 几分钟一次 |   *     | 否       | [地址](http://www.xicidaili.com)|
-  | 全网代理 |  可用  | 几分钟一次 |   *     |  否      | [地址](http://www.goubanjia.com/)|
-  | ~~训代理~~ |  已关闭免费代理  | * |   *     |  否      | [地址](http://www.xdaili.cn/)|
-  | 快代理 |  可用  |几分钟一次|   *     |  否      | [地址](https://www.kuaidaili.com/)|
-  | 云代理 |  可用  |几分钟一次|   *     |  否      | [地址](http://www.ip3366.net/)|
-  | IP海 |  可用  |几小时一次|   *     |  否      | [地址](http://www.iphai.com/)|
-  | 免费IP代理库 |  可用  |快|   *     |  否      | [地址](http://ip.jiangxianli.com/)|
-  | 中国IP地址 |  可用  |几分钟一次|   *     |  是      | [地址](http://cn-proxy.com/)|
-  | Proxy List |  可用  |几分钟一次|   *     |  是      | [地址](https://proxy-list.org/chinese/index.php)|
-  | ProxyList+ |  可用  |几分钟一次|   *     |  是      | [地址](https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1)|
+  |   代理名称   |  状态  |  更新速度 |  可用率  |  地址 |    代码   |
+  | ---------   |  ---- | --------  | ------  | ----- |   ------- |
+  | 无忧代理     |  √    |     ★     |   *    | [地址](http://www.data5u.com/) | `freeProxy01` |
+  | 66代理      |   √    |    ★★    |   *    |  [地址](http://www.66ip.cn/) |  `freeProxy02`  |
+  | ~~西刺代理~~ | ~~已关闭~~ | ~~★~~ | ~~*~~ | ~~[地址](http://www.xicidaili.com)~~| ~~`freeProxy03`~~|
+  | 全网代理     |  √    |   ★       |   *   | [地址](http://www.goubanjia.com/)| `freeProxy04` |
+  | 快代理      |  √     |   ☆       |   *   | [地址](https://www.kuaidaili.com/)| `freeProxy05` |
+  | 代理盒子    |  √     |  ★★★      |   *   | [地址](https://proxy.coderbusy.com/)| `freeProxy06` |
+  | 云代理      |  √     |  ★        |   *   | [地址](http://www.ip3366.net/)|  `freeProxy07`   |
+  | IP海       |~~已关闭~~|~~★~~     | ~~*~~ | ~~[地址](http://www.iphai.com/)~~|~~`freeProxy08`~~|
+  | 免费代理库  |  √     |   ☆       |   *   | [地址](http://ip.jiangxianli.com/)| `freeProxy09` |
+  | 89代理     |   √    |    ☆      |   *   | [地址](http://cn-proxy.com/)| `freeProxy13` |
+  | 西拉代理    |  √     |   ★★      |   *   | [地址](https://proxy-list.org/chinese/index.php)| `freeProxy14` |
+
   
   如果还有其他好的免费代理网站, 可以在提交在[issues](https://github.com/jhao104/proxy_pool/issues/71), 下次更新时会考虑在项目中支持。
 
diff --git a/docs/changelog.rst b/docs/changelog.rst
index e31ba5cb9..34d3b9b35 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -7,6 +7,9 @@ ChangeLog
 ------------------
 
 1. Fix Bug `#493`_, 新增时区配置; (2020-08-12)
+2. 修复 **66代理** 采集; (2020-11-04)
+3. 修复 **全网代理** 采集, 解决HTML端口加密问题; (2020-11-04)
+4. 新增 **代理盒子** 免费源; (2020-11-04)
 
 .. _#493: https://github.com/jhao104/proxy_pool/issues/493
 
diff --git a/fetcher/proxyFetcher.py b/fetcher/proxyFetcher.py
index d0b42996f..2a73bcc3f 100644
--- a/fetcher/proxyFetcher.py
+++ b/fetcher/proxyFetcher.py
@@ -247,24 +247,6 @@ def freeProxy09(page_count=1):
 
     @staticmethod
     def freeProxy13(max_page=2):
-        """
-        http://www.qydaili.com/free/?action=china&page=1
-        齐云代理
-        :param max_page:
-        :return:
-        """
-        base_url = 'http://www.qydaili.com/free/?action=china&page='
-        for page in range(1, max_page + 1):
-            url = base_url + str(page)
-            r = WebRequest().get(url, timeout=10)
-            proxies = re.findall(
-                r'<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td.*?>(\d+)</td>',
-                r.text)
-            for proxy in proxies:
-                yield ':'.join(proxy)
-
-    @staticmethod
-    def freeProxy14(max_page=2):
         """
         http://www.89ip.cn/index.html
         89免费代理
@@ -282,7 +264,12 @@ def freeProxy14(max_page=2):
                 yield ':'.join(proxy)
 
     @staticmethod
-    def freeProxy15():
+    def freeProxy14():
+        """
+        http://www.xiladaili.com/
+        西拉代理
+        :return:
+        """
         urls = ['http://www.xiladaili.com/putong/',
                 "http://www.xiladaili.com/gaoni/",
                 "http://www.xiladaili.com/http/",
diff --git a/setting.py b/setting.py
index 542d5549a..3b115d738 100644
--- a/setting.py
+++ b/setting.py
@@ -46,17 +46,16 @@
 # ###### config the proxy fetch function ######
 PROXY_FETCHER = [
     "freeProxy01",
-    # "freeProxy02",
+    "freeProxy02",
     # "freeProxy03",
     "freeProxy04",
     "freeProxy05",
-    # "freeProxy06",
+    "freeProxy06",
     "freeProxy07",
     # "freeProxy08",
     "freeProxy09",
     "freeProxy13",
-    "freeProxy14",
-    "freeProxy15",
+    "freeProxy14"
 ]
 
 # ############# proxy validator #################

From 97dc41a80c8667802dd3deaaf78dcb6450f82ad9 Mon Sep 17 00:00:00 2001
From: jhao <jingh@jccfc.com>
Date: Wed, 4 Nov 2020 14:56:00 +0800
Subject: [PATCH 6/6] [update] proxy

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 56884add9..909b81743 100644
--- a/README.md
+++ b/README.md
@@ -194,12 +194,12 @@ PROXY_FETCHER = [
   | ---------   |  ---- | --------  | ------  | ----- |   ------- |
   | 无忧代理     |  √    |     ★     |   *    | [地址](http://www.data5u.com/) | `freeProxy01` |
   | 66代理      |   √    |    ★★    |   *    |  [地址](http://www.66ip.cn/) |  `freeProxy02`  |
-  | ~~西刺代理~~ | ~~已关闭~~ | ~~★~~ | ~~*~~ | ~~[地址](http://www.xicidaili.com)~~| ~~`freeProxy03`~~|
+  | ~~西刺代理~~ | ~~已关闭~~ | ——   |   ——   | ~~[地址](http://www.xicidaili.com)~~| ~~`freeProxy03`~~|
   | 全网代理     |  √    |   ★       |   *   | [地址](http://www.goubanjia.com/)| `freeProxy04` |
   | 快代理      |  √     |   ☆       |   *   | [地址](https://www.kuaidaili.com/)| `freeProxy05` |
   | 代理盒子    |  √     |  ★★★      |   *   | [地址](https://proxy.coderbusy.com/)| `freeProxy06` |
   | 云代理      |  √     |  ★        |   *   | [地址](http://www.ip3366.net/)|  `freeProxy07`   |
-  | IP海       |~~已关闭~~|~~★~~     | ~~*~~ | ~~[地址](http://www.iphai.com/)~~|~~`freeProxy08`~~|
+  | ~~IP海~~   |~~已关闭~~|  ——      |   ——  | ~~[地址](http://www.iphai.com/)~~|~~`freeProxy08`~~|
   | 免费代理库  |  √     |   ☆       |   *   | [地址](http://ip.jiangxianli.com/)| `freeProxy09` |
   | 89代理     |   √    |    ☆      |   *   | [地址](http://cn-proxy.com/)| `freeProxy13` |
   | 西拉代理    |  √     |   ★★      |   *   | [地址](https://proxy-list.org/chinese/index.php)| `freeProxy14` |