From e927b91dd9dfa4ff3cb79cc196dc768af7cf6fc1 Mon Sep 17 00:00:00 2001 From: Micheal X Date: Tue, 22 Aug 2023 11:52:21 +1200 Subject: [PATCH 1/4] add browser popup log. --- code/default/launcher/start.py | 4 +++- code/default/launcher/win_tray.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/code/default/launcher/start.py b/code/default/launcher/start.py index 572980bc88..6473134470 100644 --- a/code/default/launcher/start.py +++ b/code/default/launcher/start.py @@ -165,7 +165,9 @@ def main(): if has_desktop and config.popup_webui == 1 and not restart_from_except and not no_popup: host_port = config.control_port import webbrowser - webbrowser.open("http://localhost:%s/" % host_port) + url = "http://localhost:%s/" % host_port + xlog.debug("Popup %s on startup", url) + webbrowser.open(url) if has_desktop: download_modules.start_download() diff --git a/code/default/launcher/win_tray.py b/code/default/launcher/win_tray.py index 15c932efae..854566e5ef 100644 --- a/code/default/launcher/win_tray.py +++ b/code/default/launcher/win_tray.py @@ -163,7 +163,9 @@ def on_disable_proxy(self, widget=None, data=None): def show_control_web(self, widget=None, data=None): host_port = config.control_port - webbrowser.open("http://127.0.0.1:%s/" % host_port) + url = "http://127.0.0.1:%s/" % host_port + xlog.debug("Popup %s by tray", url) + webbrowser.open(url) ctypes.windll.user32.ShowWindow(ctypes.windll.kernel32.GetConsoleWindow(), 0) def on_quit(self, widget=None, data=None): From abf94fabe080b05ca2e0872b3c7818ca49598dd5 Mon Sep 17 00:00:00 2001 From: Micheal X Date: Sun, 27 Aug 2023 17:41:15 +1200 Subject: [PATCH 2/4] not works --- code/default/gae_proxy/server/gae/app.yaml | 33 +- code/default/gae_proxy/server/gae/legacy.py | 14 - code/default/gae_proxy/server/gae/main.py | 531 ++++++++++++++++++ .../gae_proxy/server/gae/requirements.txt | 2 + code/default/gae_proxy/server/gae/wsgi.py | 293 ---------- 5 files changed, 535 insertions(+), 338 deletions(-) delete mode 100644 code/default/gae_proxy/server/gae/legacy.py create mode 100644 code/default/gae_proxy/server/gae/main.py create mode 100644 code/default/gae_proxy/server/gae/requirements.txt delete mode 100644 code/default/gae_proxy/server/gae/wsgi.py diff --git a/code/default/gae_proxy/server/gae/app.yaml b/code/default/gae_proxy/server/gae/app.yaml index c7511e8a9f..364cef1de2 100644 --- a/code/default/gae_proxy/server/gae/app.yaml +++ b/code/default/gae_proxy/server/gae/app.yaml @@ -2,35 +2,6 @@ instance_class: F1 automatic_scaling: max_instances: 1 -runtime: python27 -api_version: 1 -threadsafe: true +runtime: python311 -handlers: -- url: /_gh/.* - script: gae.application - secure: optional - -- url: /2 - script: wsgi.gae_application - secure: optional - -- url: /tasks/reset - script: gae.reset - secure: optional - -- url: /traffic - script: gae.traffic - secure: optional - -- url: /favicon.ico - script: gae.application - secure: optional - -- url: /.* - script: legacy.application - secure: optional - -libraries: -- name: pycrypto - version: "latest" +# entrypoint: gunicorn -b :$PORT main:app diff --git a/code/default/gae_proxy/server/gae/legacy.py b/code/default/gae_proxy/server/gae/legacy.py deleted file mode 100644 index 5c7e46e0e1..0000000000 --- a/code/default/gae_proxy/server/gae/legacy.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -# coding:utf-8 - -import time -from gae import __version__ - -def application(environ, start_response): - start_response('200 OK', [('Content-Type', 'text/plain; charset=UTF-8')]) - if environ['PATH_INFO'] == '/robots.txt': - yield '\n'.join(['User-agent: *', 'Disallow: /']) - else: - timestamp = long(environ['CURRENT_VERSION_ID'].split('.')[1])/2**28 - ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(timestamp+8*3600)) - yield "GoAgent 服务端已经在 %s 升级到 %s 版本, 请更新您的客户端。" % (ctime, __version__) diff --git a/code/default/gae_proxy/server/gae/main.py b/code/default/gae_proxy/server/gae/main.py new file mode 100644 index 0000000000..10ed385b27 --- /dev/null +++ b/code/default/gae_proxy/server/gae/main.py @@ -0,0 +1,531 @@ +import datetime + +from flask import Flask, render_template + + +import os +import re +import time +from datetime import timedelta, datetime, tzinfo +import struct +import zlib +import base64 +import logging +from urllib.parse import urlparse +import http.client as httplib +import io +import string +import traceback +from mimetypes import guess_type + +from Crypto.Cipher.ARC4 import new as RC4Cipher + +from google.appengine.api import wrap_wsgi_app +from google.appengine.ext import blobstore + +app = Flask(__name__) +app.wsgi_app = wrap_wsgi_app(app.wsgi_app) + + +__password__ = '' +__hostsdeny__ = () + +URLFETCH_MAX = 2 +URLFETCH_MAXSIZE = 4 * 1024 * 1024 +URLFETCH_DEFLATE_MAXSIZE = 4 * 1024 * 1024 +URLFETCH_TIMEOUT = 30 +allowed_traffic = 1024 * 1024 * 1024 * 0.9 + + +def message_html(title, banner, detail=''): + MESSAGE_TEMPLATE = ''' + + + $title + + + + + +
Message From FetchServer
+
+

$banner

+ $detail +

+

+
+ + ''' + return string.Template(MESSAGE_TEMPLATE).substitute(title=title, banner=banner, detail=detail) + + +def inflate(data): + return zlib.decompress(data, -zlib.MAX_WBITS) + + +def deflate(data): + return zlib.compress(data)[2:-4] + + +def format_response(status, headers, content): + if content: + headers.pop('content-length', None) + headers['Content-Length'] = str(len(content)) + data = 'HTTP/1.1 %d %s\r\n%s\r\n\r\n%s' % \ + (status, + httplib.responses.get(status, 'Unknown'), + '\r\n'.join('%s: %s' % (k.title(), v) for k, v in headers.items()), + content) + data = deflate(data) + return struct.pack('!h', len(data)) + data + + +def is_text_content_type(content_type): + mct, _, sct = content_type.partition('/') + if mct == 'text': + return True + if mct == 'application': + sct = sct.split(';', 1)[0] + if (sct in ('json', 'javascript', 'x-www-form-urlencoded') or + sct.endswith(('xml', 'script')) or + sct.startswith(('xml', 'rss', 'atom'))): + return True + return False + + +def is_deflate(data): + if len(data) > 1: + CMF, FLG = bytearray(data[:2]) + if CMF & 0x0F == 8 and CMF & 0x80 == 0 and ((CMF << 8) + FLG) % 31 == 0: + return True + if len(data) > 0: + try: + decompressor = zlib.decompressobj(-zlib.MAX_WBITS) + decompressor.decompress(data[:1024]) + return decompressor.unused_data == '' + except: + return False + return False + + +class Pacific(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=-8) + self.dst(dt) + + def dst(self, dt): + # DST starts last Sunday in March + d = datetime(dt.year, 3, 12) # ends last Sunday in October + self.dston = d - timedelta(days=d.weekday() + 1) + d = datetime(dt.year, 11, 6) + self.dstoff = d - timedelta(days=d.weekday() + 1) + if self.dston <= dt.replace(tzinfo=None) < self.dstoff: + return timedelta(hours=1) + else: + return timedelta(0) + + def tzname(self,dt): + return "Pacific" + + +def get_pacific_date(): + tz = Pacific() + sa_time = datetime.now(tz) + return sa_time.strftime('%Y-%m-%d') + + +def traffic(environ, start_response): + try: + # reset_date = memcache.get(key="reset_date") + reset_date = blobstore.get("reset_date") + except: + reset_date = None + + try: + # traffic_sum = memcache.get(key="traffic") + traffic_sum = blobstore.get("traffic") + if not traffic_sum: + traffic_sum = "0" + except Exception as e: + traffic_sum = "0" + + start_response('200 OK', [('Content-Type', 'text/plain')]) + yield 'traffic:%s\r\n' % traffic_sum + yield 'Reset date:%s\r\n' % reset_date + yield 'Usage: %f %%\r\n' % int(int(traffic_sum) * 100 / allowed_traffic) + + tz = Pacific() + sa_time = datetime.now(tz) + pacific_time = sa_time.strftime('%Y-%m-%d %H:%M:%S') + yield "American Pacific time:%s" % pacific_time + + raise StopIteration + + +def reset(environ, start_response): + try: + # memcache.set(key="traffic", value="0") + blobstore.set("traffic", "0") + except: + pass + + start_response('200 OK', [('Content-Type', 'text/plain')]) + yield 'traffic reset finished.' + raise StopIteration + + +def is_traffic_exceed(): + try: + # reset_date = memcache.get(key="reset_date") + reset_date = blobstore.get("reset_date") + except: + reset_date = None + + pacific_date = get_pacific_date() + if reset_date != pacific_date: + # memcache.set(key="reset_date", value=pacific_date) + # memcache.set(key="traffic", value="0") + blobstore.set("reset_date", pacific_date) + blobstore.set("traffic", "0") + return False + + try: + # traffic_sum = int(memcache.get(key="traffic")) + traffic_sum = int(blobstore.get("traffic")) + except: + traffic_sum = 0 + + if traffic_sum > allowed_traffic: + return True + else: + return False + + +def count_traffic(add_traffic): + try: + # traffic_sum = int(memcache.get(key="traffic")) + traffic_sum = int(blobstore.get("traffic")) + except: + traffic_sum = 0 + + try: + v = str(traffic_sum + add_traffic) + # memcache.set(key="traffic", value=v) + blobstore.set("traffic", v) + except Exception as e: + logging.exception('memcache.set fail:%r', e) + + + +@app.route("/") +def root(): + out = "GoAgent 服务端已经升级到 python3 版本。
\nVersion: 4.0.0" + return out + + +@app.route("/_gh/", methods=['POST']) +def proxy(): + + start_response('200 OK', [('Content-Type', 'image/gif'), ('X-Server', 'GPS ' + __version__)]) + + + options = environ.get('HTTP_X_URLFETCH_OPTIONS', '') + # 不知道怎么直接获得的 + # 但一般，此段语句无用 + if 'rc4' in options and not __password__: + # 如果客户端需要加密，但ｇａｅ无密码 + + # 但rc4 如不改源码，则恒为假 + yield format_response(400, + {'Content-Type': 'text/html; charset=utf-8'}, + message_html('400 Bad Request', + 'Bad Request (options) - please set __password__ in gae.py', + 'please set __password__ and upload gae.py again')) + raise StopIteration + + try: + if 'HTTP_X_URLFETCH_PS1' in environ: + # 第一部分 + payload = inflate(base64.b64decode(environ['HTTP_X_URLFETCH_PS1'])) + body = inflate( + base64.b64decode( + # 第二部分　即原始ｂｏｄｙ + environ['HTTP_X_URLFETCH_PS2'])) if 'HTTP_X_URLFETCH_PS2' in environ else '' + else: + # POST + # POST 获取数据的方式 + wsgi_input = environ['wsgi.input'] + input_data = wsgi_input.read(int(environ.get('CONTENT_LENGTH', '0'))) + + if 'rc4' in options: + input_data = RC4Cipher(__password__).encrypt(input_data) + payload_length, = struct.unpack('!h', input_data[:2]) # 获取长度 + payload = inflate(input_data[2:2 + payload_length]) # 获取负载 + body = input_data[2 + payload_length:] # 获取ｂｏｄｙ + + count_traffic(len(input_data)) + raw_response_line, payload = payload.split('\r\n', 1) + method, url = raw_response_line.split()[:2] + # http content: + # 此为ｂｏｄｙ + # { + # pack_req_head_len: 2 bytes,＃ＰＯＳＴ　时使用 + + # pack_req_head : deflate{ + # 此为负载 + # original request line, + # original request headers, + # X-URLFETCH-kwargs HEADS, { + # password, + # maxsize, defined in config AUTO RANGE MAX SIZE + # timeout, request timeout for GAE urlfetch. + # } + # } + # body + # } + + headers = {} + # 获取　原始头 + for line in payload.splitlines(): + key, value = line.split(':', 1) + headers[key.title()] = value.strip() + except (zlib.error, KeyError, ValueError): + yield format_response(500, + {'Content-Type': 'text/html; charset=utf-8'}, + message_html('500 Internal Server Error', + 'Bad Request (payload) - Possible Wrong Password', + '
%s
' % traceback.format_exc())) + raise StopIteration + + # 获取ｇａｅ用的头 + kwargs = {} + any(kwargs.__setitem__(x[len('x-urlfetch-'):].lower(), headers.pop(x)) for x in headers.keys() if + x.lower().startswith('x-urlfetch-')) + + if 'Content-Encoding' in headers and body: + # fix bug for LinkedIn android client + if headers['Content-Encoding'] == 'deflate': + try: + body2 = inflate(body) + headers['Content-Length'] = str(len(body2)) + del headers['Content-Encoding'] + body = body2 + except BaseException: + pass + + ref = headers.get('Referer', '') + logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, ref) + + # 参数使用 + if __password__ and __password__ != kwargs.get('password', ''): + yield format_response(401, {'Content-Type': 'text/html; charset=utf-8'}, + message_html('403 Wrong password', 'Wrong password(%r)' % kwargs.get('password', ''), + 'GoAgent proxy.ini password is wrong!')) + raise StopIteration + + netloc = urlparse.urlparse(url).netloc + if is_traffic_exceed(): + yield format_response(510, {'Content-Type': 'text/html; charset=utf-8'}, + message_html('510 Traffic exceed', + 'Traffic exceed', + 'Traffic exceed!')) + raise StopIteration + + if len(url) > MAX_URL_LENGTH: + yield format_response(400, + {'Content-Type': 'text/html; charset=utf-8'}, + message_html('400 Bad Request', + 'length of URL too long(greater than %r)' % MAX_URL_LENGTH, + detail='url=%r' % url)) + raise StopIteration + + if netloc.startswith(('127.0.0.', '::1', 'localhost')): + # 测试用 + yield format_response(400, {'Content-Type': 'text/html; charset=utf-8'}, + message_html('GoAgent %s is Running' % __version__, 'Now you can visit some websites', + ''.join('%s
' % (x, x) for x in + ('google.com', 'mail.google.com')))) + raise StopIteration + + fetchmethod = getattr(urlfetch, method, None) + if not fetchmethod: + yield format_response(405, {'Content-Type': 'text/html; charset=utf-8'}, + message_html('405 Method Not Allowed', 'Method Not Allowed: %r' % method, + detail='Method Not Allowed URL=%r' % url)) + raise StopIteration + + timeout = int(kwargs.get('timeout', URLFETCH_TIMEOUT)) + validate_certificate = bool(int(kwargs.get('validate', 0))) + maxsize = int(kwargs.get('maxsize', 0)) + # https://www.freebsdchina.org/forum/viewtopic.php?t=54269 + accept_encoding = headers.get('Accept-Encoding', '') or headers.get('Bccept-Encoding', '') + errors = [] + allow_truncated = False + for i in xrange(int(kwargs.get('fetchmax', URLFETCH_MAX))): + try: + response = urlfetch.fetch( + url, + body, + fetchmethod, + headers, + allow_truncated=allow_truncated, + follow_redirects=False, + deadline=timeout, + validate_certificate=validate_certificate) + # 获取真正ｒｅｓｐｏｎｓｅ + break + except apiproxy_errors.OverQuotaError as e: + time.sleep(5) + except urlfetch.DeadlineExceededError as e: + errors.append('%r, timeout=%s' % (e, timeout)) + logging.error( + 'DeadlineExceededError(timeout=%s, url=%r)', + timeout, + url) + time.sleep(1) + + # 必须truncaated + allow_truncated = True + m = re.search(r'=\s*(\d+)-', headers.get('Range') + or headers.get('range') or '') + if m is None: + headers['Range'] = 'bytes=0-%d' % (maxsize or URLFETCH_MAXSIZE) + else: + headers.pop('Range', '') + headers.pop('range', '') + start = int(m.group(1)) + headers['Range'] = 'bytes=%s-%d' % (start, + start + (maxsize or URLFETCH_MAXSIZE)) + + timeout *= 2 + except urlfetch.DownloadError as e: + errors.append('%r, timeout=%s' % (e, timeout)) + logging.error('DownloadError(timeout=%s, url=%r)', timeout, url) + time.sleep(1) + timeout *= 2 + except urlfetch.ResponseTooLargeError as e: + errors.append('%r, timeout=%s' % (e, timeout)) + response = e.response + logging.error( + 'ResponseTooLargeError(timeout=%s, url=%r) response(%r)', + timeout, + url, + response) + + m = re.search(r'=\s*(\d+)-', headers.get('Range') + or headers.get('range') or '') + if m is None: + headers['Range'] = 'bytes=0-%d' % (maxsize or URLFETCH_MAXSIZE) + else: + headers.pop('Range', '') + headers.pop('range', '') + start = int(m.group(1)) + headers['Range'] = 'bytes=%s-%d' % (start, + start + (maxsize or URLFETCH_MAXSIZE)) + timeout *= 2 + except urlfetch.SSLCertificateError as e: + errors.append('%r, should validate=0 ?' % e) + logging.error('%r, timeout=%s', e, timeout) + except Exception as e: + errors.append(str(e)) + stack_str = "stack:%s" % traceback.format_exc() + errors.append(stack_str) + if i == 0 and method == 'GET': + timeout *= 2 + else: + error_string = '
\n'.join(errors) + if not error_string: + logurl = 'https://appengine.google.com/logs?&app_id=%s' % os.environ['APPLICATION_ID'] + error_string = 'Internal Server Error.

try refresh' \ + ' or goto appengine.google.com for details' % logurl + yield format_response(502, {'Content-Type': 'text/html; charset=utf-8'}, + message_html('502 Urlfetch Error', 'Python Urlfetch Error: %r' % method, error_string)) + raise StopIteration + + # logging.debug('url=%r response.status_code=%r response.headers=%r response.content[:1024]=%r', url, + # response.status_code, dict(response.headers), response.content[:1024]) + + # 以上实现ｆｅｔｃｈ 的细节 + + status_code = int(response.status_code) + data = response.content + response_headers = response.headers + response_headers['X-Head-Content-Length'] = response_headers.get( + 'Content-Length', '') + # for k in response_headers: + # v = response_headers[k] + # logging.debug("Head:%s: %s", k, v) + content_type = response_headers.get('content-type', '') + content_encoding = response_headers.get('content-encoding', '') + # 也是分片合并之类的细节 + if status_code == 200 and maxsize and len(data) > maxsize and response_headers.get( + 'accept-ranges', '').lower() == 'bytes' and int(response_headers.get('content-length', 0)): + logging.debug("data len:%d max:%d", len(data), maxsize) + status_code = 206 + response_headers['Content-Range'] = 'bytes 0-%d/%d' % ( + maxsize - 1, len(data)) + data = data[:maxsize] + if 'gzip' in accept_encoding: + if (data and status_code == 200 and + content_encoding == '' and + is_text_content_type(content_type) and + is_deflate(data)): + # ignore wrong "Content-Type" + type = guess_type(url)[0] + if type is None or is_text_content_type(type): + if 'deflate' in accept_encoding: + response_headers['Content-Encoding'] = content_encoding = 'deflate' + else: + data = inflate(data) + else: + if content_encoding in ('gzip', 'deflate', 'br'): + del response_headers['Content-Encoding'] + content_encoding = '' + if status_code == 200 and content_encoding == '' and 512 < len( + data) < URLFETCH_DEFLATE_MAXSIZE and is_text_content_type(content_type): + if 'gzip' in accept_encoding: + response_headers['Content-Encoding'] = 'gzip' + compressobj = zlib.compressobj( + zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) + dataio = io.BytesIO() + dataio.write('\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff') + dataio.write(compressobj.compress(data)) + dataio.write(compressobj.flush()) + dataio.write( + struct.pack( + ' - -__version__ = '3.0.7' -__password__ = '' -from gae import __hostsdeny__ -__content_type__ = 'image/gif' - -import sys -import os -import re -import time -import struct -import zlib -import base64 -import logging -import httplib -import urlparse -import errno -import string - - - -try: - from io import BytesIO -except ImportError: - from cStringIO import StringIO as BytesIO -try: - from google.appengine.api import urlfetch - from google.appengine.runtime import apiproxy_errors -except ImportError: - urlfetch = None -try: - import sae -except ImportError: - sae = None -try: - import bae.core.wsgi -except ImportError: - bae = None -try: - import socket - import select -except ImportError: - socket = None -try: - import OpenSSL -except ImportError: - OpenSSL = None - -URLFETCH_MAX = 2 -URLFETCH_MAXSIZE = 4*1024*1024 -URLFETCH_DEFLATE_MAXSIZE = 4*1024*1024 -URLFETCH_TIMEOUT = 60 - -def message_html(title, banner, detail=''): - MESSAGE_TEMPLATE = ''' - - - $title - - - - - -
Message
-

-

$banner

- $detail -

-

-
- - ''' - return string.Template(MESSAGE_TEMPLATE).substitute(title=title, banner=banner, detail=detail) - - -try: - from Crypto.Cipher.ARC4 import new as _Crypto_Cipher_ARC4_new -except ImportError: - logging.warn('Load Crypto.Cipher.ARC4 Failed, Use Pure Python Instead.') - class _Crypto_Cipher_ARC4_new(object): - def __init__(self, key): - x = 0 - box = range(256) - for i, y in enumerate(box): - x = (x + y + ord(key[i % len(key)])) & 0xff - box[i], box[x] = box[x], y - self.__box = box - self.__x = 0 - self.__y = 0 - def encrypt(self, data): - out = [] - out_append = out.append - x = self.__x - y = self.__y - box = self.__box - for char in data: - x = (x + 1) & 0xff - y = (y + box[x]) & 0xff - box[x], box[y] = box[y], box[x] - out_append(chr(ord(char) ^ box[(box[x] + box[y]) & 0xff])) - self.__x = x - self.__y = y - return ''.join(out) - - -def rc4crypt(data, key): - return _Crypto_Cipher_ARC4_new(key).encrypt(data) if key else data - - -class RC4FileObject(object): - """fileobj for rc4""" - def __init__(self, stream, key): - self.__stream = stream - self.__cipher = _Crypto_Cipher_ARC4_new(key) if key else lambda x:x - def __getattr__(self, attr): - if attr not in ('__stream', '__cipher'): - return getattr(self.__stream, attr) - def read(self, size=-1): - return self.__cipher.encrypt(self.__stream.read(size)) - - -def gae_application(environ, start_response): - cookie = environ.get('HTTP_COOKIE', '') - options = environ.get('HTTP_X_GOA_OPTIONS', '') - if environ['REQUEST_METHOD'] == 'GET' and not cookie: - if '204' in environ['QUERY_STRING']: - start_response('204 No Content', []) - yield '' - else: - timestamp = long(os.environ['CURRENT_VERSION_ID'].split('.')[1])/2**28 - ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(timestamp+8*3600)) - html = u'GoAgent Python Server %s \u5df2\u7ecf\u5728\u5de5\u4f5c\u4e86\uff0c\u90e8\u7f72\u65f6\u95f4 %s\n' % (__version__, ctime) - start_response('200 OK', [('Content-Type', 'text/plain; charset=utf-8')]) - yield html.encode('utf8') - raise StopIteration - - # inflate = lambda x:zlib.decompress(x, -zlib.MAX_WBITS) - wsgi_input = environ['wsgi.input'] - input_data = wsgi_input.read() - - try: - if cookie: - if 'rc4' not in options: - metadata = zlib.decompress(base64.b64decode(cookie), -zlib.MAX_WBITS) - payload = input_data or '' - else: - metadata = zlib.decompress(rc4crypt(base64.b64decode(cookie), __password__), -zlib.MAX_WBITS) - payload = rc4crypt(input_data, __password__) if input_data else '' - else: - if 'rc4' in options: - input_data = rc4crypt(input_data, __password__) - metadata_length, = struct.unpack('!h', input_data[:2]) - metadata = zlib.decompress(input_data[2:2+metadata_length], -zlib.MAX_WBITS) - payload = input_data[2+metadata_length:] - headers = dict(x.split(':', 1) for x in metadata.splitlines() if x) - method = headers.pop('G-Method') - url = headers.pop('G-Url') - except (zlib.error, KeyError, ValueError): - import traceback - start_response('500 Internal Server Error', [('Content-Type', 'text/html')]) - yield message_html('500 Internal Server Error', 'Bad Request (metadata) - Possible Wrong Password', '
%s
' % traceback.format_exc()) - raise StopIteration - - kwargs = {} - any(kwargs.__setitem__(x[2:].lower(), headers.pop(x)) for x in headers.keys() if x.startswith('G-')) - - if 'Content-Encoding' in headers: - if headers['Content-Encoding'] == 'deflate': - payload = zlib.decompress(payload, -zlib.MAX_WBITS) - headers['Content-Length'] = str(len(payload)) - del headers['Content-Encoding'] - - ref = headers.get('Referer', '') - logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, ref) - #logging.info('request headers=%s', headers) - - if __password__ and __password__ != kwargs.get('password', ''): - start_response('403 Forbidden', [('Content-Type', 'text/html')]) - yield message_html('403 Wrong password', 'Wrong password(%r)' % kwargs.get('password', ''), 'GoAgent proxy.ini password is wrong!') - raise StopIteration - - netloc = urlparse.urlparse(url).netloc - - if __hostsdeny__ and netloc.endswith(__hostsdeny__): - start_response('403 Forbidden', [('Content-Type', 'text/html')]) - yield message_html('403 Hosts Deny', 'Hosts Deny(%r)' % netloc, detail='共用appid因为资源有限，限制观看视频和文件下载等消耗资源过多的访问，请使用自己的appid 帮助 ') - raise StopIteration - - if netloc.startswith(('127.0.0.', '::1', 'localhost')): - start_response('400 Bad Request', [('Content-Type', 'text/html')]) - html = ''.join('%s
' % (x, x) for x in ('google.com', 'mail.google.com')) - yield message_html('GoAgent %s is Running' % __version__, 'Now you can visit some websites', html) - raise StopIteration - - fetchmethod = getattr(urlfetch, method, None) - if not fetchmethod: - start_response('405 Method Not Allowed', [('Content-Type', 'text/html')]) - yield message_html('405 Method Not Allowed', 'Method Not Allowed: %r' % method, detail='Method Not Allowed URL=%r' % url) - raise StopIteration - - deadline = URLFETCH_TIMEOUT - validate_certificate = bool(int(kwargs.get('validate', 0))) - accept_encoding = headers.get('Accept-Encoding', '') - errors = [] - for i in xrange(int(kwargs.get('fetchmax', URLFETCH_MAX))): - try: - response = urlfetch.fetch(url, payload, fetchmethod, headers, allow_truncated=False, follow_redirects=False, deadline=deadline, validate_certificate=validate_certificate) - break - except apiproxy_errors.OverQuotaError as e: - time.sleep(5) - except urlfetch.DeadlineExceededError as e: - errors.append('%r, deadline=%s' % (e, deadline)) - logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url) - time.sleep(1) - deadline = URLFETCH_TIMEOUT * 2 - except urlfetch.DownloadError as e: - errors.append('%r, deadline=%s' % (e, deadline)) - logging.error('DownloadError(deadline=%s, url=%r)', deadline, url) - time.sleep(1) - deadline = URLFETCH_TIMEOUT * 2 - except urlfetch.ResponseTooLargeError as e: - errors.append('%r, deadline=%s' % (e, deadline)) - response = e.response - logging.error('ResponseTooLargeError(deadline=%s, url=%r) response(%r)', deadline, url, response) - m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '') - if m is None: - headers['Range'] = 'bytes=0-%d' % int(kwargs.get('fetchmaxsize', URLFETCH_MAXSIZE)) - else: - headers.pop('Range', '') - headers.pop('range', '') - start = int(m.group(1)) - headers['Range'] = 'bytes=%s-%d' % (start, start+int(kwargs.get('fetchmaxsize', URLFETCH_MAXSIZE))) - deadline = URLFETCH_TIMEOUT * 2 - except urlfetch.SSLCertificateError as e: - errors.append('%r, should validate=0 ?' % e) - logging.error('%r, deadline=%s', e, deadline) - except Exception as e: - errors.append(str(e)) - if i == 0 and method == 'GET': - deadline = URLFETCH_TIMEOUT * 2 - else: - start_response('500 Internal Server Error', [('Content-Type', 'text/html')]) - error_string = '
\n'.join(errors) - if not error_string: - logurl = 'https://appengine.google.com/logs?&app_id=%s' % os.environ['APPLICATION_ID'] - error_string = 'Internal Server Error.

try refresh or goto appengine.google.com for details' % logurl - yield message_html('502 Urlfetch Error', 'Python Urlfetch Error: %r' % method, error_string) - raise StopIteration - - #logging.debug('url=%r response.status_code=%r response.headers=%r response.content[:1024]=%r', url, response.status_code, dict(response.headers), response.content[:1024]) - - data = response.content - response_headers = response.headers - if 'content-encoding' not in response_headers and len(response.content) < URLFETCH_DEFLATE_MAXSIZE and response_headers.get('content-type', '').startswith(('text/', 'application/json', 'application/javascript')): - if 'gzip' in accept_encoding: - response_headers['Content-Encoding'] = 'gzip' - compressobj = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) - dataio = BytesIO() - dataio.write('\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff') - dataio.write(compressobj.compress(data)) - dataio.write(compressobj.flush()) - dataio.write(struct.pack(' Date: Fri, 1 Dec 2023 16:15:48 +1300 Subject: [PATCH 3/4] upgrade server to python3. --- code/default/gae_proxy/local/config.py | 2 +- code/default/gae_proxy/server/README.md | 5 +- .../gae_proxy/server/gae/.gcloudignore | 19 + code/default/gae_proxy/server/gae/app.yaml | 7 +- code/default/gae_proxy/server/gae/gae.py | 562 --------------- .../gae_proxy/server/gae/gunicorn.conf.py | 5 + code/default/gae_proxy/server/gae/main.py | 671 +++++++----------- .../gae_proxy/server/gae/requirements.txt | 6 +- code/default/gae_proxy/tests/test_protocol.py | 289 ++++++++ 9 files changed, 567 insertions(+), 999 deletions(-) create mode 100644 code/default/gae_proxy/server/gae/.gcloudignore delete mode 100644 code/default/gae_proxy/server/gae/gae.py create mode 100644 code/default/gae_proxy/server/gae/gunicorn.conf.py create mode 100644 code/default/gae_proxy/tests/test_protocol.py diff --git a/code/default/gae_proxy/local/config.py b/code/default/gae_proxy/local/config.py index 1bb7fc854e..007ad99ca9 100644 --- a/code/default/gae_proxy/local/config.py +++ b/code/default/gae_proxy/local/config.py @@ -36,7 +36,7 @@ def __init__(self, fn): # gae self.set_var("GAE_PASSWORD", "") - self.set_var("GAE_VALIDATE", 0) + self.set_var("GAE_VALIDATE", 1) # host rules self.set_var("hosts_direct", [ diff --git a/code/default/gae_proxy/server/README.md b/code/default/gae_proxy/server/README.md index 9f1bfa31a8..d361bcd069 100644 --- a/code/default/gae_proxy/server/README.md +++ b/code/default/gae_proxy/server/README.md @@ -1,6 +1,6 @@ ## 设置使用密码： 如果你担心流量被别人使用，可以设置使用密码。 -编辑gae.py 里的 __password__ +编辑main.py 里的 __password____ 注意在客户端中也需要设置一样的密码才能访问。 一般你不泄露appid，别人是无法使用你的流量的。 @@ -21,6 +21,9 @@ https://cloud.google.com/sdk/docs/install `gcloud init` +## 未绑定信用卡的需要绑定: + https://console.cloud.google.com/billing/enable + ## 部署： `cd XX-Net/code/default/gae_proxy/server/gae` `gcloud app deploy` diff --git a/code/default/gae_proxy/server/gae/.gcloudignore b/code/default/gae_proxy/server/gae/.gcloudignore new file mode 100644 index 0000000000..603f0b6ea0 --- /dev/null +++ b/code/default/gae_proxy/server/gae/.gcloudignore @@ -0,0 +1,19 @@ +# This file specifies files that are *not* uploaded to Google Cloud +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +# Python pycache: +__pycache__/ +# Ignored by the build system +/setup.cfg \ No newline at end of file diff --git a/code/default/gae_proxy/server/gae/app.yaml b/code/default/gae_proxy/server/gae/app.yaml index 364cef1de2..2d537bf4e2 100644 --- a/code/default/gae_proxy/server/gae/app.yaml +++ b/code/default/gae_proxy/server/gae/app.yaml @@ -2,6 +2,9 @@ instance_class: F1 automatic_scaling: max_instances: 1 -runtime: python311 +runtime: python312 -# entrypoint: gunicorn -b :$PORT main:app +entrypoint: gunicorn -c gunicorn.conf.py --timeout 60 -b :$PORT main:app +#handlers: +# - url: /.* +# script: auto diff --git a/code/default/gae_proxy/server/gae/gae.py b/code/default/gae_proxy/server/gae/gae.py deleted file mode 100644 index 490373e980..0000000000 --- a/code/default/gae_proxy/server/gae/gae.py +++ /dev/null @@ -1,562 +0,0 @@ -#!/usr/bin/env python -# coding:utf-8 - - -# GAE limit: -# only support http/https request, don't support tcp/udp connect for unpaid user. -# max timeout for every request is 60 seconds -# max upload data size is 30M -# max download data size is 10M - -# How to Download file large then 10M? -# HTTP protocol support range fetch. -# If server return header include "accept-ranges", then client can request special range -# by put Content-Range in request header. -# -# GAE server will return 206 status code if file is too large and server support range fetch. -# Then GAE_proxy local client will switch to range fetch mode. - - -__version__ = '3.4.0' -__password__ = '' -__hostsdeny__ = () - -import os -import re -import time -from datetime import timedelta, datetime, tzinfo -import struct -import zlib -import base64 -import logging -import urlparse -import httplib -import io -import string -import traceback -from mimetypes import guess_type - -from google.appengine.api import urlfetch -from google.appengine.api.taskqueue.taskqueue import MAX_URL_LENGTH -from google.appengine.runtime import apiproxy_errors -from google.appengine.api import memcache - -URLFETCH_MAX = 2 -URLFETCH_MAXSIZE = 4 * 1024 * 1024 -URLFETCH_DEFLATE_MAXSIZE = 4 * 1024 * 1024 -URLFETCH_TIMEOUT = 30 -allowed_traffic = 1024 * 1024 * 1024 * 0.9 - - -def message_html(title, banner, detail=''): - MESSAGE_TEMPLATE = ''' - - - $title - - - - - -
Message From FetchServer
-

-

$banner

- $detail -

-

-
- - ''' - return string.Template(MESSAGE_TEMPLATE).substitute(title=title, banner=banner, detail=detail) - - -try: - from Crypto.Cipher.ARC4 import new as RC4Cipher -except ImportError: - logging.warn('Load Crypto.Cipher.ARC4 Failed, Use Pure Python Instead.') - - - class RC4Cipher(object): - def __init__(self, key): - x = 0 - box = range(256) - for i, y in enumerate(box): - x = (x + y + ord(key[i % len(key)])) & 0xff - box[i], box[x] = box[x], y - self.__box = box - self.__x = 0 - self.__y = 0 - - def encrypt(self, data): - out = [] - out_append = out.append - x = self.__x - y = self.__y - box = self.__box - for char in data: - x = (x + 1) & 0xff - y = (y + box[x]) & 0xff - box[x], box[y] = box[y], box[x] - out_append(chr(ord(char) ^ box[(box[x] + box[y]) & 0xff])) - self.__x = x - self.__y = y - return ''.join(out) - - -def inflate(data): - return zlib.decompress(data, -zlib.MAX_WBITS) - - -def deflate(data): - return zlib.compress(data)[2:-4] - - -def format_response(status, headers, content): - if content: - headers.pop('content-length', None) - headers['Content-Length'] = str(len(content)) - data = 'HTTP/1.1 %d %s\r\n%s\r\n\r\n%s' % \ - (status, - httplib.responses.get(status, 'Unknown'), - '\r\n'.join('%s: %s' % (k.title(), v) for k, v in headers.items()), - content) - data = deflate(data) - return struct.pack('!h', len(data)) + data - - -def is_text_content_type(content_type): - mct, _, sct = content_type.partition('/') - if mct == 'text': - return True - if mct == 'application': - sct = sct.split(';', 1)[0] - if (sct in ('json', 'javascript', 'x-www-form-urlencoded') or - sct.endswith(('xml', 'script')) or - sct.startswith(('xml', 'rss', 'atom'))): - return True - return False - - -def is_deflate(data): - if len(data) > 1: - CMF, FLG = bytearray(data[:2]) - if CMF & 0x0F == 8 and CMF & 0x80 == 0 and ((CMF << 8) + FLG) % 31 == 0: - return True - if len(data) > 0: - try: - decompressor = zlib.decompressobj(-zlib.MAX_WBITS) - decompressor.decompress(data[:1024]) - return decompressor.unused_data == '' - except: - return False - return False - - -class Pacific(tzinfo): - def utcoffset(self, dt): - return timedelta(hours=-8) + self.dst(dt) - - def dst(self, dt): - # DST starts last Sunday in March - d = datetime(dt.year, 3, 12) # ends last Sunday in October - self.dston = d - timedelta(days=d.weekday() + 1) - d = datetime(dt.year, 11, 6) - self.dstoff = d - timedelta(days=d.weekday() + 1) - if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - return timedelta(hours=1) - else: - return timedelta(0) - - def tzname(self,dt): - return "Pacific" - - -def get_pacific_date(): - tz = Pacific() - sa_time = datetime.now(tz) - return sa_time.strftime('%Y-%m-%d') - - -def traffic(environ, start_response): - try: - reset_date = memcache.get(key="reset_date") - except: - reset_date = None - - try: - traffic_sum = memcache.get(key="traffic") - if not traffic_sum: - traffic_sum = "0" - except Exception as e: - traffic_sum = "0" - - start_response('200 OK', [('Content-Type', 'text/plain')]) - yield 'traffic:%s\r\n' % traffic_sum - yield 'Reset date:%s\r\n' % reset_date - yield 'Usage: %f %%\r\n' % int(int(traffic_sum) * 100 / allowed_traffic) - - tz = Pacific() - sa_time = datetime.now(tz) - pacific_time = sa_time.strftime('%Y-%m-%d %H:%M:%S') - yield "American Pacific time:%s" % pacific_time - - raise StopIteration - - -def reset(environ, start_response): - try: - memcache.set(key="traffic", value="0") - except: - pass - - start_response('200 OK', [('Content-Type', 'text/plain')]) - yield 'traffic reset finished.' - raise StopIteration - - -def is_traffic_exceed(): - try: - reset_date = memcache.get(key="reset_date") - except: - reset_date = None - - pacific_date = get_pacific_date() - if reset_date != pacific_date: - memcache.set(key="reset_date", value=pacific_date) - memcache.set(key="traffic", value="0") - return False - - try: - traffic_sum = int(memcache.get(key="traffic")) - except: - traffic_sum = 0 - - if traffic_sum > allowed_traffic: - return True - else: - return False - - -def count_traffic(add_traffic): - try: - traffic_sum = int(memcache.get(key="traffic")) - except: - traffic_sum = 0 - - try: - v = str(traffic_sum + add_traffic) - memcache.set(key="traffic", value=v) - except Exception as e: - logging.exception('memcache.set fail:%r', e) - - -def application(environ, start_response): - if environ['REQUEST_METHOD'] == 'GET' and 'HTTP_X_URLFETCH_PS1' not in environ: - # xxnet 自用 - timestamp = long(os.environ['CURRENT_VERSION_ID'].split('.')[1]) / 2 ** 28 - ctime = time.strftime( - '%Y-%m-%d %H:%M:%S', - time.gmtime( - timestamp + 8 * 3600)) - start_response('200 OK', [('Content-Type', 'text/plain')]) - yield 'GoAgent Python Server %s works, deployed at %s\n' % (__version__, ctime) - if len(__password__) > 2: - yield 'Password: %s%s%s' % (__password__[0], '*' * (len(__password__) - 2), __password__[-1]) - raise StopIteration - - start_response('200 OK', [('Content-Type', 'image/gif'), ('X-Server', 'GPS ' + __version__)]) - - if environ['REQUEST_METHOD'] == 'HEAD': - raise StopIteration - # 请求头则已经完成 - - options = environ.get('HTTP_X_URLFETCH_OPTIONS', '') - # 不知道怎么直接获得的 - # 但一般，此段语句无用 - if 'rc4' in options and not __password__: - # 如果客户端需要加密，但ｇａｅ无密码 - - # 但rc4 如不改源码，则恒为假 - yield format_response(400, - {'Content-Type': 'text/html; charset=utf-8'}, - message_html('400 Bad Request', - 'Bad Request (options) - please set __password__ in gae.py', - 'please set __password__ and upload gae.py again')) - raise StopIteration - - try: - if 'HTTP_X_URLFETCH_PS1' in environ: - # 第一部分 - payload = inflate(base64.b64decode(environ['HTTP_X_URLFETCH_PS1'])) - body = inflate( - base64.b64decode( - # 第二部分　即原始ｂｏｄｙ - environ['HTTP_X_URLFETCH_PS2'])) if 'HTTP_X_URLFETCH_PS2' in environ else '' - else: - # POST - # POST 获取数据的方式 - wsgi_input = environ['wsgi.input'] - input_data = wsgi_input.read(int(environ.get('CONTENT_LENGTH', '0'))) - - if 'rc4' in options: - input_data = RC4Cipher(__password__).encrypt(input_data) - payload_length, = struct.unpack('!h', input_data[:2]) # 获取长度 - payload = inflate(input_data[2:2 + payload_length]) # 获取负载 - body = input_data[2 + payload_length:] # 获取ｂｏｄｙ - - count_traffic(len(input_data)) - raw_response_line, payload = payload.split('\r\n', 1) - method, url = raw_response_line.split()[:2] - # http content: - # 此为ｂｏｄｙ - # { - # pack_req_head_len: 2 bytes,＃ＰＯＳＴ　时使用 - - # pack_req_head : deflate{ - # 此为负载 - # original request line, - # original request headers, - # X-URLFETCH-kwargs HEADS, { - # password, - # maxsize, defined in config AUTO RANGE MAX SIZE - # timeout, request timeout for GAE urlfetch. - # } - # } - # body - # } - - headers = {} - # 获取　原始头 - for line in payload.splitlines(): - key, value = line.split(':', 1) - headers[key.title()] = value.strip() - except (zlib.error, KeyError, ValueError): - yield format_response(500, - {'Content-Type': 'text/html; charset=utf-8'}, - message_html('500 Internal Server Error', - 'Bad Request (payload) - Possible Wrong Password', - '
%s
' % traceback.format_exc())) - raise StopIteration - - # 获取ｇａｅ用的头 - kwargs = {} - any(kwargs.__setitem__(x[len('x-urlfetch-'):].lower(), headers.pop(x)) for x in headers.keys() if - x.lower().startswith('x-urlfetch-')) - - if 'Content-Encoding' in headers and body: - # fix bug for LinkedIn android client - if headers['Content-Encoding'] == 'deflate': - try: - body2 = inflate(body) - headers['Content-Length'] = str(len(body2)) - del headers['Content-Encoding'] - body = body2 - except BaseException: - pass - - ref = headers.get('Referer', '') - logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, ref) - - # 参数使用 - if __password__ and __password__ != kwargs.get('password', ''): - yield format_response(401, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('403 Wrong password', 'Wrong password(%r)' % kwargs.get('password', ''), - 'GoAgent proxy.ini password is wrong!')) - raise StopIteration - - netloc = urlparse.urlparse(url).netloc - if is_traffic_exceed(): - yield format_response(510, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('510 Traffic exceed', - 'Traffic exceed', - 'Traffic exceed!')) - raise StopIteration - - if len(url) > MAX_URL_LENGTH: - yield format_response(400, - {'Content-Type': 'text/html; charset=utf-8'}, - message_html('400 Bad Request', - 'length of URL too long(greater than %r)' % MAX_URL_LENGTH, - detail='url=%r' % url)) - raise StopIteration - - if netloc.startswith(('127.0.0.', '::1', 'localhost')): - # 测试用 - yield format_response(400, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('GoAgent %s is Running' % __version__, 'Now you can visit some websites', - ''.join('%s
' % (x, x) for x in - ('google.com', 'mail.google.com')))) - raise StopIteration - - fetchmethod = getattr(urlfetch, method, None) - if not fetchmethod: - yield format_response(405, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('405 Method Not Allowed', 'Method Not Allowed: %r' % method, - detail='Method Not Allowed URL=%r' % url)) - raise StopIteration - - timeout = int(kwargs.get('timeout', URLFETCH_TIMEOUT)) - validate_certificate = bool(int(kwargs.get('validate', 0))) - maxsize = int(kwargs.get('maxsize', 0)) - # https://www.freebsdchina.org/forum/viewtopic.php?t=54269 - accept_encoding = headers.get('Accept-Encoding', '') or headers.get('Bccept-Encoding', '') - errors = [] - allow_truncated = False - for i in xrange(int(kwargs.get('fetchmax', URLFETCH_MAX))): - try: - response = urlfetch.fetch( - url, - body, - fetchmethod, - headers, - allow_truncated=allow_truncated, - follow_redirects=False, - deadline=timeout, - validate_certificate=validate_certificate) - # 获取真正ｒｅｓｐｏｎｓｅ - break - except apiproxy_errors.OverQuotaError as e: - time.sleep(5) - except urlfetch.DeadlineExceededError as e: - errors.append('%r, timeout=%s' % (e, timeout)) - logging.error( - 'DeadlineExceededError(timeout=%s, url=%r)', - timeout, - url) - time.sleep(1) - - # 必须truncaated - allow_truncated = True - m = re.search(r'=\s*(\d+)-', headers.get('Range') - or headers.get('range') or '') - if m is None: - headers['Range'] = 'bytes=0-%d' % (maxsize or URLFETCH_MAXSIZE) - else: - headers.pop('Range', '') - headers.pop('range', '') - start = int(m.group(1)) - headers['Range'] = 'bytes=%s-%d' % (start, - start + (maxsize or URLFETCH_MAXSIZE)) - - timeout *= 2 - except urlfetch.DownloadError as e: - errors.append('%r, timeout=%s' % (e, timeout)) - logging.error('DownloadError(timeout=%s, url=%r)', timeout, url) - time.sleep(1) - timeout *= 2 - except urlfetch.ResponseTooLargeError as e: - errors.append('%r, timeout=%s' % (e, timeout)) - response = e.response - logging.error( - 'ResponseTooLargeError(timeout=%s, url=%r) response(%r)', - timeout, - url, - response) - - m = re.search(r'=\s*(\d+)-', headers.get('Range') - or headers.get('range') or '') - if m is None: - headers['Range'] = 'bytes=0-%d' % (maxsize or URLFETCH_MAXSIZE) - else: - headers.pop('Range', '') - headers.pop('range', '') - start = int(m.group(1)) - headers['Range'] = 'bytes=%s-%d' % (start, - start + (maxsize or URLFETCH_MAXSIZE)) - timeout *= 2 - except urlfetch.SSLCertificateError as e: - errors.append('%r, should validate=0 ?' % e) - logging.error('%r, timeout=%s', e, timeout) - except Exception as e: - errors.append(str(e)) - stack_str = "stack:%s" % traceback.format_exc() - errors.append(stack_str) - if i == 0 and method == 'GET': - timeout *= 2 - else: - error_string = '
\n'.join(errors) - if not error_string: - logurl = 'https://appengine.google.com/logs?&app_id=%s' % os.environ['APPLICATION_ID'] - error_string = 'Internal Server Error.

try refresh' \ - ' or goto appengine.google.com for details' % logurl - yield format_response(502, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('502 Urlfetch Error', 'Python Urlfetch Error: %r' % method, error_string)) - raise StopIteration - - # logging.debug('url=%r response.status_code=%r response.headers=%r response.content[:1024]=%r', url, - # response.status_code, dict(response.headers), response.content[:1024]) - - # 以上实现ｆｅｔｃｈ 的细节 - - status_code = int(response.status_code) - data = response.content - response_headers = response.headers - response_headers['X-Head-Content-Length'] = response_headers.get( - 'Content-Length', '') - # for k in response_headers: - # v = response_headers[k] - # logging.debug("Head:%s: %s", k, v) - content_type = response_headers.get('content-type', '') - content_encoding = response_headers.get('content-encoding', '') - # 也是分片合并之类的细节 - if status_code == 200 and maxsize and len(data) > maxsize and response_headers.get( - 'accept-ranges', '').lower() == 'bytes' and int(response_headers.get('content-length', 0)): - logging.debug("data len:%d max:%d", len(data), maxsize) - status_code = 206 - response_headers['Content-Range'] = 'bytes 0-%d/%d' % ( - maxsize - 1, len(data)) - data = data[:maxsize] - if 'gzip' in accept_encoding: - if (data and status_code == 200 and - content_encoding == '' and - is_text_content_type(content_type) and - is_deflate(data)): - # ignore wrong "Content-Type" - type = guess_type(url)[0] - if type is None or is_text_content_type(type): - if 'deflate' in accept_encoding: - response_headers['Content-Encoding'] = content_encoding = 'deflate' - else: - data = inflate(data) - else: - if content_encoding in ('gzip', 'deflate', 'br'): - del response_headers['Content-Encoding'] - content_encoding = '' - if status_code == 200 and content_encoding == '' and 512 < len( - data) < URLFETCH_DEFLATE_MAXSIZE and is_text_content_type(content_type): - if 'gzip' in accept_encoding: - response_headers['Content-Encoding'] = 'gzip' - compressobj = zlib.compressobj( - zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) - dataio = io.BytesIO() - dataio.write('\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff') - dataio.write(compressobj.compress(data)) - dataio.write(compressobj.flush()) - dataio.write( - struct.pack( - ' - - $title - - - - - -
Message From FetchServer
-

-

$banner

- $detail -

-

-
- - ''' - return string.Template(MESSAGE_TEMPLATE).substitute(title=title, banner=banner, detail=detail) +def map_with_parameter(function, datas, args): + plist = [] + for data in datas: + d_out = function(data, args) + plist.append(d_out) + return plist + + +def to_bytes(data, coding='utf-8'): + if isinstance(data, bytes): + return data + if isinstance(data, str): + return data.encode(coding) + if isinstance(data, dict): + return dict(map_with_parameter(to_bytes, data.items(), coding)) + if isinstance(data, tuple): + return tuple(map_with_parameter(to_bytes, data, coding)) + if isinstance(data, list): + return list(map_with_parameter(to_bytes, data, coding)) + if isinstance(data, int): + return to_bytes(str(data)) + if data is None: + return data + return bytes(data) + + +def to_str(data, coding='utf-8'): + if isinstance(data, str): + return data + if isinstance(data, bytes): + return data.decode(coding) + if isinstance(data, bytearray): + return data.decode(coding) + if isinstance(data, dict): + return dict(map_with_parameter(to_str, data.items(), coding)) + if isinstance(data, tuple): + return tuple(map_with_parameter(to_str, data, coding)) + if isinstance(data, list): + return list(map_with_parameter(to_str, data, coding)) + if isinstance(data, int): + return str(data) + if data is None: + return data + return str(data) def inflate(data): @@ -76,45 +81,54 @@ def deflate(data): return zlib.compress(data)[2:-4] -def format_response(status, headers, content): - if content: +def unpack_request(payload): + head_len = struct.unpack('!h', payload[0:2])[0] + head = payload[2:2+head_len] + body = payload[2+head_len:] + + head = inflate(head) + lines = head.split(b"\r\n") + method, url = lines[0].split()[:2] + headers = {} + kwargs = {} + for line in lines[1:]: + ls = line.split(b": ") + k = ls[0] + if not k: + continue + + v = b"".join(ls[1:]) + if k.startswith(b"X-URLFETCH-"): + k = k[11:] + kwargs[k] = v + else: + headers[k] = v + + timeout = int(kwargs.get(b"timeout", 30)) + if headers.get(b"Content-Encoding") == b"deflate": + body = inflate(body) + del headers[b"Content-Encoding"] + + return method, url, headers, body, timeout, kwargs + + +def pack_response(status, headers, app_msg, content): + if app_msg: headers.pop('content-length', None) - headers['Content-Length'] = str(len(content)) - data = 'HTTP/1.1 %d %s\r\n%s\r\n\r\n%s' % \ - (status, - httplib.responses.get(status, 'Unknown'), - '\r\n'.join('%s: %s' % (k.title(), v) for k, v in headers.items()), - content) - data = deflate(data) - return struct.pack('!h', len(data)) + data - - -def is_text_content_type(content_type): - mct, _, sct = content_type.partition('/') - if mct == 'text': - return True - if mct == 'application': - sct = sct.split(';', 1)[0] - if (sct in ('json', 'javascript', 'x-www-form-urlencoded') or - sct.endswith(('xml', 'script')) or - sct.startswith(('xml', 'rss', 'atom'))): - return True - return False + headers['Content-Length'] = str(len(app_msg)) + headers = to_bytes(headers) -def is_deflate(data): - if len(data) > 1: - CMF, FLG = bytearray(data[:2]) - if CMF & 0x0F == 8 and CMF & 0x80 == 0 and ((CMF << 8) + FLG) % 31 == 0: - return True - if len(data) > 0: - try: - decompressor = zlib.decompressobj(-zlib.MAX_WBITS) - decompressor.decompress(data[:1024]) - return decompressor.unused_data == '' - except: - return False - return False + data = b'HTTP/1.1 %d %s\r\n%s\r\n\r\n%s' % \ + (status, + to_bytes(httplib.responses.get(status, 'Unknown')), + b'\r\n'.join(b'%s: %s' % (k.title(), v) for k, v in headers.items()), + app_msg) + + data = deflate(data) + head_len_pack = struct.pack('!h', len(data)) + out = head_len_pack + data + to_bytes(content) + return out class Pacific(tzinfo): @@ -142,87 +156,88 @@ def get_pacific_date(): return sa_time.strftime('%Y-%m-%d') -def traffic(environ, start_response): - try: - # reset_date = memcache.get(key="reset_date") - reset_date = blobstore.get("reset_date") - except: - reset_date = None - - try: - # traffic_sum = memcache.get(key="traffic") - traffic_sum = blobstore.get("traffic") - if not traffic_sum: - traffic_sum = "0" - except Exception as e: - traffic_sum = "0" - - start_response('200 OK', [('Content-Type', 'text/plain')]) - yield 'traffic:%s\r\n' % traffic_sum - yield 'Reset date:%s\r\n' % reset_date - yield 'Usage: %f %%\r\n' % int(int(traffic_sum) * 100 / allowed_traffic) - - tz = Pacific() - sa_time = datetime.now(tz) - pacific_time = sa_time.strftime('%Y-%m-%d %H:%M:%S') - yield "American Pacific time:%s" % pacific_time - - raise StopIteration - - -def reset(environ, start_response): - try: - # memcache.set(key="traffic", value="0") - blobstore.set("traffic", "0") - except: - pass - - start_response('200 OK', [('Content-Type', 'text/plain')]) - yield 'traffic reset finished.' - raise StopIteration +# def traffic(environ, start_response): +# try: +# # reset_date = memcache.get(key="reset_date") +# reset_date = blobstore.get("reset_date") +# except: +# reset_date = None +# +# try: +# # traffic_sum = memcache.get(key="traffic") +# traffic_sum = blobstore.get("traffic") +# if not traffic_sum: +# traffic_sum = "0" +# except Exception as e: +# traffic_sum = "0" +# +# start_response('200 OK', [('Content-Type', 'text/plain')]) +# yield 'traffic:%s\r\n' % traffic_sum +# yield 'Reset date:%s\r\n' % reset_date +# yield 'Usage: %f %%\r\n' % int(int(traffic_sum) * 100 / allowed_traffic) +# +# tz = Pacific() +# sa_time = datetime.now(tz) +# pacific_time = sa_time.strftime('%Y-%m-%d %H:%M:%S') +# yield "American Pacific time:%s" % pacific_time +# +# raise StopIteration + + +# def reset(environ, start_response): +# try: +# # memcache.set(key="traffic", value="0") +# blobstore.set("traffic", "0") +# except: +# pass +# +# start_response('200 OK', [('Content-Type', 'text/plain')]) +# yield 'traffic reset finished.' +# raise StopIteration def is_traffic_exceed(): - try: - # reset_date = memcache.get(key="reset_date") - reset_date = blobstore.get("reset_date") - except: - reset_date = None - - pacific_date = get_pacific_date() - if reset_date != pacific_date: - # memcache.set(key="reset_date", value=pacific_date) - # memcache.set(key="traffic", value="0") - blobstore.set("reset_date", pacific_date) - blobstore.set("traffic", "0") - return False - - try: - # traffic_sum = int(memcache.get(key="traffic")) - traffic_sum = int(blobstore.get("traffic")) - except: - traffic_sum = 0 - - if traffic_sum > allowed_traffic: - return True - else: - return False + return False +# try: +# # reset_date = memcache.get(key="reset_date") +# reset_date = blobstore.get("reset_date") +# except: +# reset_date = None +# +# pacific_date = get_pacific_date() +# if reset_date != pacific_date: +# # memcache.set(key="reset_date", value=pacific_date) +# # memcache.set(key="traffic", value="0") +# blobstore.set("reset_date", pacific_date) +# blobstore.set("traffic", "0") +# return False +# +# try: +# # traffic_sum = int(memcache.get(key="traffic")) +# traffic_sum = int(blobstore.get("traffic")) +# except: +# traffic_sum = 0 +# +# if traffic_sum > allowed_traffic: +# return True +# else: +# return False def count_traffic(add_traffic): - try: - # traffic_sum = int(memcache.get(key="traffic")) - traffic_sum = int(blobstore.get("traffic")) - except: - traffic_sum = 0 - - try: - v = str(traffic_sum + add_traffic) - # memcache.set(key="traffic", value=v) - blobstore.set("traffic", v) - except Exception as e: - logging.exception('memcache.set fail:%r', e) - + pass +# try: +# # traffic_sum = int(memcache.get(key="traffic")) +# traffic_sum = int(blobstore.get("traffic")) +# except: +# traffic_sum = 0 +# +# try: +# v = str(traffic_sum + add_traffic) +# # memcache.set(key="traffic", value=v) +# blobstore.set("traffic", v) +# except Exception as e: +# logging.exception('memcache.set fail:%r', e) @app.route("/") @@ -231,293 +246,87 @@ def root(): return out -@app.route("/_gh/", methods=['POST']) -def proxy(): +@app.route("/_gh/", methods=['GET']) +def check(): + logging.debug("req headers:%s", request.headers) + return "GoAgent works" - start_response('200 OK', [('Content-Type', 'image/gif'), ('X-Server', 'GPS ' + __version__)]) +def req_by_requests(method, url, req_headers, req_body, timeout, verify, kwargs): + # maxsize = int(kwargs.get('maxsize', 0)) + # accept_encoding = headers.get('Accept-Encoding', '') - options = environ.get('HTTP_X_URLFETCH_OPTIONS', '') - # 不知道怎么直接获得的 - # 但一般，此段语句无用 - if 'rc4' in options and not __password__: - # 如果客户端需要加密，但ｇａｅ无密码 + errors = [] + for i in range(int(kwargs.get('fetchmax', URLFETCH_MAX))): + try: + res = requests.request(method, url, headers=req_headers, data=req_body, timeout=timeout, verify=verify, + stream=True, allow_redirects=False) + break + except Exception as e: + logging.warning("request %s %s %s %s %s e:%r", method, url, req_headers, timeout, verify, e) + errors.append(str(e)) + if i == 0 and method == 'GET': + timeout *= 2 + else: + error_string = '
\n'.join(errors) + logging.info('%s "%s %s" error:%s', request.remote_addr, method, url, error_string) + return 502, {}, "502 Urlfetch Error: " + error_string - # 但rc4 如不改源码，则恒为假 - yield format_response(400, - {'Content-Type': 'text/html; charset=utf-8'}, - message_html('400 Bad Request', - 'Bad Request (options) - please set __password__ in gae.py', - 'please set __password__ and upload gae.py again')) - raise StopIteration + res_headers = dict(res.headers) + res_content = res.raw.read() + # logging.debug(f'url={url} status_code={res.status_code} headers={res_headers} content={len(res_content)}') + if "Transfer-Encoding" in res_headers: + del res_headers["Transfer-Encoding"] + + res_headers["X-Head-Content-Length"] = res_headers["Content-Length"] = len(res_content) + return res.status_code, res_headers, res_content + + +@app.route("/_gh/", methods=['POST']) +def proxy(): + t0 = time.time() try: - if 'HTTP_X_URLFETCH_PS1' in environ: - # 第一部分 - payload = inflate(base64.b64decode(environ['HTTP_X_URLFETCH_PS1'])) - body = inflate( - base64.b64decode( - # 第二部分　即原始ｂｏｄｙ - environ['HTTP_X_URLFETCH_PS2'])) if 'HTTP_X_URLFETCH_PS2' in environ else '' - else: - # POST - # POST 获取数据的方式 - wsgi_input = environ['wsgi.input'] - input_data = wsgi_input.read(int(environ.get('CONTENT_LENGTH', '0'))) - - if 'rc4' in options: - input_data = RC4Cipher(__password__).encrypt(input_data) - payload_length, = struct.unpack('!h', input_data[:2]) # 获取长度 - payload = inflate(input_data[2:2 + payload_length]) # 获取负载 - body = input_data[2 + payload_length:] # 获取ｂｏｄｙ - - count_traffic(len(input_data)) - raw_response_line, payload = payload.split('\r\n', 1) - method, url = raw_response_line.split()[:2] - # http content: - # 此为ｂｏｄｙ - # { - # pack_req_head_len: 2 bytes,＃ＰＯＳＴ　时使用 - - # pack_req_head : deflate{ - # 此为负载 - # original request line, - # original request headers, - # X-URLFETCH-kwargs HEADS, { - # password, - # maxsize, defined in config AUTO RANGE MAX SIZE - # timeout, request timeout for GAE urlfetch. - # } - # } - # body - # } - - headers = {} - # 获取　原始头 - for line in payload.splitlines(): - key, value = line.split(':', 1) - headers[key.title()] = value.strip() - except (zlib.error, KeyError, ValueError): - yield format_response(500, - {'Content-Type': 'text/html; charset=utf-8'}, - message_html('500 Internal Server Error', - 'Bad Request (payload) - Possible Wrong Password', - '
%s
' % traceback.format_exc())) - raise StopIteration - - # 获取ｇａｅ用的头 - kwargs = {} - any(kwargs.__setitem__(x[len('x-urlfetch-'):].lower(), headers.pop(x)) for x in headers.keys() if - x.lower().startswith('x-urlfetch-')) - - if 'Content-Encoding' in headers and body: - # fix bug for LinkedIn android client - if headers['Content-Encoding'] == 'deflate': - try: - body2 = inflate(body) - headers['Content-Length'] = str(len(body2)) - del headers['Content-Encoding'] - body = body2 - except BaseException: - pass - - ref = headers.get('Referer', '') - logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, ref) + method, url, req_headers, req_body, timeout, kwargs = unpack_request(request.data) + method = to_str(method) + url = to_str(url) + req_headers = to_str(req_headers) + kwargs = to_str(kwargs) + except Exception as e: + logging.exception("unpack request:%r", e) + return "500 Bad Request", 500 + + # logging.info('from:%s method:%s url:%s kwargs:%s -', request.remote_addr, method, url, kwargs) # 参数使用 - if __password__ and __password__ != kwargs.get('password', ''): - yield format_response(401, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('403 Wrong password', 'Wrong password(%r)' % kwargs.get('password', ''), - 'GoAgent proxy.ini password is wrong!')) - raise StopIteration + if __password__ != kwargs.get('password', ''): + logging.info('wrong password') + return "401 Wrong password", 401 - netloc = urlparse.urlparse(url).netloc if is_traffic_exceed(): - yield format_response(510, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('510 Traffic exceed', - 'Traffic exceed', - 'Traffic exceed!')) - raise StopIteration - - if len(url) > MAX_URL_LENGTH: - yield format_response(400, - {'Content-Type': 'text/html; charset=utf-8'}, - message_html('400 Bad Request', - 'length of URL too long(greater than %r)' % MAX_URL_LENGTH, - detail='url=%r' % url)) - raise StopIteration + logging.info('Traffic exceed') + return "510 Traffic exceed", 510 + netloc = urlparse(url).netloc if netloc.startswith(('127.0.0.', '::1', 'localhost')): - # 测试用 - yield format_response(400, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('GoAgent %s is Running' % __version__, 'Now you can visit some websites', - ''.join('%s
' % (x, x) for x in - ('google.com', 'mail.google.com')))) - raise StopIteration - - fetchmethod = getattr(urlfetch, method, None) - if not fetchmethod: - yield format_response(405, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('405 Method Not Allowed', 'Method Not Allowed: %r' % method, - detail='Method Not Allowed URL=%r' % url)) - raise StopIteration + return "GoAgent is Running", 400 timeout = int(kwargs.get('timeout', URLFETCH_TIMEOUT)) - validate_certificate = bool(int(kwargs.get('validate', 0))) - maxsize = int(kwargs.get('maxsize', 0)) - # https://www.freebsdchina.org/forum/viewtopic.php?t=54269 - accept_encoding = headers.get('Accept-Encoding', '') or headers.get('Bccept-Encoding', '') - errors = [] - allow_truncated = False - for i in xrange(int(kwargs.get('fetchmax', URLFETCH_MAX))): - try: - response = urlfetch.fetch( - url, - body, - fetchmethod, - headers, - allow_truncated=allow_truncated, - follow_redirects=False, - deadline=timeout, - validate_certificate=validate_certificate) - # 获取真正ｒｅｓｐｏｎｓｅ - break - except apiproxy_errors.OverQuotaError as e: - time.sleep(5) - except urlfetch.DeadlineExceededError as e: - errors.append('%r, timeout=%s' % (e, timeout)) - logging.error( - 'DeadlineExceededError(timeout=%s, url=%r)', - timeout, - url) - time.sleep(1) - - # 必须truncaated - allow_truncated = True - m = re.search(r'=\s*(\d+)-', headers.get('Range') - or headers.get('range') or '') - if m is None: - headers['Range'] = 'bytes=0-%d' % (maxsize or URLFETCH_MAXSIZE) - else: - headers.pop('Range', '') - headers.pop('range', '') - start = int(m.group(1)) - headers['Range'] = 'bytes=%s-%d' % (start, - start + (maxsize or URLFETCH_MAXSIZE)) - - timeout *= 2 - except urlfetch.DownloadError as e: - errors.append('%r, timeout=%s' % (e, timeout)) - logging.error('DownloadError(timeout=%s, url=%r)', timeout, url) - time.sleep(1) - timeout *= 2 - except urlfetch.ResponseTooLargeError as e: - errors.append('%r, timeout=%s' % (e, timeout)) - response = e.response - logging.error( - 'ResponseTooLargeError(timeout=%s, url=%r) response(%r)', - timeout, - url, - response) - - m = re.search(r'=\s*(\d+)-', headers.get('Range') - or headers.get('range') or '') - if m is None: - headers['Range'] = 'bytes=0-%d' % (maxsize or URLFETCH_MAXSIZE) - else: - headers.pop('Range', '') - headers.pop('range', '') - start = int(m.group(1)) - headers['Range'] = 'bytes=%s-%d' % (start, - start + (maxsize or URLFETCH_MAXSIZE)) - timeout *= 2 - except urlfetch.SSLCertificateError as e: - errors.append('%r, should validate=0 ?' % e) - logging.error('%r, timeout=%s', e, timeout) - except Exception as e: - errors.append(str(e)) - stack_str = "stack:%s" % traceback.format_exc() - errors.append(stack_str) - if i == 0 and method == 'GET': - timeout *= 2 - else: - error_string = '
\n'.join(errors) - if not error_string: - logurl = 'https://appengine.google.com/logs?&app_id=%s' % os.environ['APPLICATION_ID'] - error_string = 'Internal Server Error.

try refresh' \ - ' or goto appengine.google.com for details' % logurl - yield format_response(502, {'Content-Type': 'text/html; charset=utf-8'}, - message_html('502 Urlfetch Error', 'Python Urlfetch Error: %r' % method, error_string)) - raise StopIteration - - # logging.debug('url=%r response.status_code=%r response.headers=%r response.content[:1024]=%r', url, - # response.status_code, dict(response.headers), response.content[:1024]) - - # 以上实现ｆｅｔｃｈ 的细节 - - status_code = int(response.status_code) - data = response.content - response_headers = response.headers - response_headers['X-Head-Content-Length'] = response_headers.get( - 'Content-Length', '') - # for k in response_headers: - # v = response_headers[k] - # logging.debug("Head:%s: %s", k, v) - content_type = response_headers.get('content-type', '') - content_encoding = response_headers.get('content-encoding', '') - # 也是分片合并之类的细节 - if status_code == 200 and maxsize and len(data) > maxsize and response_headers.get( - 'accept-ranges', '').lower() == 'bytes' and int(response_headers.get('content-length', 0)): - logging.debug("data len:%d max:%d", len(data), maxsize) - status_code = 206 - response_headers['Content-Range'] = 'bytes 0-%d/%d' % ( - maxsize - 1, len(data)) - data = data[:maxsize] - if 'gzip' in accept_encoding: - if (data and status_code == 200 and - content_encoding == '' and - is_text_content_type(content_type) and - is_deflate(data)): - # ignore wrong "Content-Type" - type = guess_type(url)[0] - if type is None or is_text_content_type(type): - if 'deflate' in accept_encoding: - response_headers['Content-Encoding'] = content_encoding = 'deflate' - else: - data = inflate(data) - else: - if content_encoding in ('gzip', 'deflate', 'br'): - del response_headers['Content-Encoding'] - content_encoding = '' - if status_code == 200 and content_encoding == '' and 512 < len( - data) < URLFETCH_DEFLATE_MAXSIZE and is_text_content_type(content_type): - if 'gzip' in accept_encoding: - response_headers['Content-Encoding'] = 'gzip' - compressobj = zlib.compressobj( - zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) - dataio = io.BytesIO() - dataio.write('\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff') - dataio.write(compressobj.compress(data)) - dataio.write(compressobj.flush()) - dataio.write( - struct.pack( - ' 10 * 1024 * 1024: + xlog.warn("body len:%d %s %s", len(body), method, url) + headers[b'Content-Length'] = utils.to_bytes(str(len(body))) + + # GAE don't allow set `Host` header + if b'Host' in headers: + del headers[b'Host'] + + kwargs = {} + # gae 用的参数 + if config.GAE_PASSWORD: + kwargs[b'password'] = config.GAE_PASSWORD + + # kwargs['options'] = + kwargs[b'validate'] = config.GAE_VALIDATE + if url.endswith(b".js"): + kwargs[b'maxsize'] = config.JS_MAXSIZE + else: + kwargs[b'maxsize'] = config.AUTORANGE_MAXSIZE + kwargs[b'timeout'] = str(timeout) + # gae 用的参数　ｅｎｄ + + payload = b'%s %s HTTP/1.1\r\n' % (method, url) + payload += b''.join(b'%s: %s\r\n' % (k, v) + for k, v in list(headers.items()) if k not in skip_request_headers) + # for k, v in headers.items(): + # xlog.debug("Send %s: %s", k, v) + for k, v in kwargs.items(): + if isinstance(v, int): + payload += b'X-URLFETCH-%s: %d\r\n' % (k, v) + else: + payload += b'X-URLFETCH-%s: %s\r\n' % (k, utils.to_bytes(v)) + + payload = deflate(payload) + + body = b'%s%s%s' % (struct.pack('!h', len(payload)), payload, body) + request_headers = {} + request_headers[b'Content-Length'] = str(len(body)) + # request_headers 只有上面一项 + + return request_headers, body + + +def unpack_request(payload): + head_len = struct.unpack('!h', payload[0:2])[0] + print(head_len) + head = payload[2:2+head_len] + body = payload[2+head_len:] + + head = inflate(head) + lines = head.split(b"\r\n") + method, url = lines[0].split()[:2] + headers = {} + kwargs = {} + for line in lines[1:]: + ls = line.split(b": ") + k = ls[0] + if not k: + continue + + v = b"".join(ls[1:]) + if k.startswith(b"X-URLFETCH-"): + k = k[11:] + kwargs[k] = v + else: + headers[k] = v + + timeout = int(kwargs.get(b"timeout", 30)) + if headers.get(b"Content-Encoding") == b"deflate": + body = inflate(body) + del headers[b"Content-Encoding"] + + return method, url, headers, body, timeout, kwargs + + +def pack_response(status, headers, app_msg, content): + if app_msg: + headers.pop('content-length', None) + headers['Content-Length'] = str(len(app_msg)) + + headers = utils.to_bytes(headers) + + data = b'HTTP/1.1 %d %s\r\n%s\r\n\r\n%s' % \ + (status, + utils.to_bytes(httplib.responses.get(status, 'Unknown')), + b'\r\n'.join(b'%s: %s' % (k.title(), v) for k, v in headers.items()), + app_msg) + data = deflate(data) + return struct.pack('!h', len(data)) + data + content + + +def unpack_response(body): + try: + data = body[:2] + if not data: + raise Exception(600, "get protocol head fail") + + if len(data) !=2: + raise Exception(600, "get protocol head fail, data:%s, len:%d" % (data, len(data))) + + headers_length, = struct.unpack('!h', data) + data = body[2:2+headers_length] + if not data: + raise Exception(600, + "get protocol head fail, len:%d" % headers_length) + + raw_response_line, headers_data = inflate(data).split(b'\r\n', 1) + rl = raw_response_line.split() + status_code = int(rl[1]) + if len(rl) >=3: + reason = rl[2].strip() + else: + reason = b"" + + headers_block, app_msg = headers_data.split(b'\r\n\r\n', 1) + headers_pairs = headers_block.split(b'\r\n') + headers = {} + for pair in headers_pairs: + if not pair: + break + k, v = pair.split(b': ', 1) + headers[k] = v + + content = body[2+headers_length:] + return status_code, reason, headers, app_msg, content + except Exception as e: + raise Exception(600, "unpack protocol:%r at:%s" % (e, traceback.format_exc())) + + +class TestProtocol(TestCase): + def test_req(self): + method = b"POST" + url = b"https://cloud.google.com/" + info = { + "req": "a", + "type": "b" + } + body = utils.to_bytes(json.dumps(info)) + headers = { + "Content-Length": str(len(body)), + "Content-Type": "application/json" + } + headers = utils.to_bytes(headers) + timeout = 30 + + request_headers, payload = pack_request(method, url, headers, body, timeout) + + method1, url1, headers1, body1, timeout1, kwargs = unpack_request(payload) + print(f"method:{method1}") + print(f"url1:{url1}") + print(f"headers1:{headers1}") + print(f"body1:{body1}") + print(f"timeout1:{timeout1}") + print(f"kwargs:{kwargs}") + + def test_response(self): + status = 200 + headers = { + b"Cookie": b"abc" + } + content = b"ABC" + payload = pack_response(status, headers, b"", content) + + status_code, reason, res_headers, app_msg, body = unpack_response(payload) + self.assertEqual(status, status_code) + # self.assertEqual(headers, res_headers) + self.assertEqual(content, body) + print(f"status:{status_code}") + print(f"reason:{reason}") + logging.debug(f"res_headers:{res_headers}") + logger.debug(f"body:{body}") + + def test_pack_real_response(self): + res = requests.get("https://github.com") + status = res.status_code + headers = dict(res.headers) + content = res.content + payload = pack_response(status, headers, b"", content) + + status_code, reason, res_headers, app_msg, body = unpack_response(payload) + self.assertEqual(status, status_code) + # self.assertEqual(headers, res_headers) + self.assertEqual(content, body) + print(f"status:{status_code}") + print(f"reason:{reason}") + print(f"res_headers:{res_headers}") + logging.debug(f"body:{body}") + + def test_req_local(self): + method = b"GET" + url = b"http://www.github.com/" + # info = { + # "User-Agent": "python" + # } + # body = utils.to_bytes(json.dumps(info)) + body = b"" + headers = { + # "Content-Length": str(len(body)), + # "Content-Type": "application/json" + "Accept-Encoding": "gzip, br" + } + headers = utils.to_bytes(headers) + timeout = 30 + + request_headers, payload = pack_request(method, url, headers, body, timeout) + + res = requests.post("http://localhost:8080/_gh/", data=payload) + + status_code, reason, res_headers, app_msg, body = unpack_response(res.content) + logging.debug(f"status:{status_code}") + logging.debug(f"reason:{reason}") + res_headers = utils.to_str(res_headers) + logging.debug(f"res_headers:{json.dumps(res_headers, indent=2)}") + logging.debug(f"body_len:{len(body)}") + logging.debug(f"body_100:{body[:100]}") From df2210ae072221177b77b096034f40998075e84f Mon Sep 17 00:00:00 2001 From: Micheal X Date: Sun, 3 Dec 2023 14:38:42 +1300 Subject: [PATCH 4/4] upgrade server to python3 tested. --- code/default/gae_proxy/server/gae/app.yaml | 6 +- .../gae_proxy/server/gae/gunicorn.conf.py | 4 +- code/default/gae_proxy/server/gae/main.py | 399 +++++++++++++----- .../gae_proxy/server/gae/requirements.txt | 4 +- code/default/gae_proxy/tests/test_protocol.py | 36 +- 5 files changed, 315 insertions(+), 134 deletions(-) diff --git a/code/default/gae_proxy/server/gae/app.yaml b/code/default/gae_proxy/server/gae/app.yaml index 2d537bf4e2..c367fa3017 100644 --- a/code/default/gae_proxy/server/gae/app.yaml +++ b/code/default/gae_proxy/server/gae/app.yaml @@ -3,8 +3,6 @@ automatic_scaling: max_instances: 1 runtime: python312 +app_engine_apis: true -entrypoint: gunicorn -c gunicorn.conf.py --timeout 60 -b :$PORT main:app -#handlers: -# - url: /.* -# script: auto +entrypoint: gunicorn -c gunicorn.conf.py --timeout 60 -b :$PORT main:app \ No newline at end of file diff --git a/code/default/gae_proxy/server/gae/gunicorn.conf.py b/code/default/gae_proxy/server/gae/gunicorn.conf.py index ce9d6aab90..825fbed502 100644 --- a/code/default/gae_proxy/server/gae/gunicorn.conf.py +++ b/code/default/gae_proxy/server/gae/gunicorn.conf.py @@ -1,5 +1,5 @@ # Recommended number of workers based on instance size: # https://cloud.google.com/appengine/docs/standard/python3/runtime#entrypoint_best_practices -workers = 1 +threads = 20 # Use an asynchronous worker as most of the work is waiting for websites to load -worker_class = 'gevent' \ No newline at end of file +worker_class = 'gthread' \ No newline at end of file diff --git a/code/default/gae_proxy/server/gae/main.py b/code/default/gae_proxy/server/gae/main.py index 1722ee15e3..c2defeb9c5 100644 --- a/code/default/gae_proxy/server/gae/main.py +++ b/code/default/gae_proxy/server/gae/main.py @@ -1,30 +1,51 @@ - +import os import time from datetime import timedelta, datetime, tzinfo import struct import zlib import logging from urllib.parse import urlparse -from urllib.request import urlopen, Request import http.client as httplib +import json +import threading +import re +import psutil import flask from flask import Flask, request import requests +from google.cloud import storage +from google.appengine.api import urlfetch +from google.appengine.runtime import apiproxy_errors +from google.appengine.api.taskqueue.taskqueue import MAX_URL_LENGTH +from google.appengine.api import urlfetch_stub +from google.appengine.api import apiproxy_stub_map + +apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() +apiproxy_stub_map.apiproxy.RegisterStub('urlfetch', urlfetch_stub.URLFetchServiceStub()) logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.INFO) app = Flask(__name__) +try: + storage_client = storage.Client() + project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") + bucket_name = f"{project_id}.appspot.com" + bucket = storage_client.bucket(bucket_name) + blob_name = "band_usage_info" +except Exception as e: + logging.warning("init blob fail:%r", e) + storage_client = None __password__ = '' URLFETCH_MAX = 2 -# URLFETCH_MAXSIZE = 4 * 1024 * 1024 -# URLFETCH_DEFLATE_MAXSIZE = 4 * 1024 * 1024 +URLFETCH_DEFLATE_MAXSIZE = 4 * 1024 * 1024 URLFETCH_TIMEOUT = 30 -allowed_traffic = 1024 * 1024 * 1024 * 0.9 +allowed_traffic = 1024 * 1024 * 1024 * 0.90 +gae_support_methods = tuple(["GET", "POST", "HEAD", "PUT", "DELETE", "PATCH"]) def map_with_parameter(function, datas, args): @@ -104,12 +125,18 @@ def unpack_request(payload): else: headers[k] = v - timeout = int(kwargs.get(b"timeout", 30)) if headers.get(b"Content-Encoding") == b"deflate": body = inflate(body) del headers[b"Content-Encoding"] - return method, url, headers, body, timeout, kwargs + method = to_str(method) + url = to_str(url) + headers = to_str(headers) + kwargs = to_str(kwargs) + if method == "GET" and "Content-Length" in headers: + del headers["Content-Length"] + + return method, url, headers, body, kwargs def pack_response(status, headers, app_msg, content): @@ -156,88 +183,104 @@ def get_pacific_date(): return sa_time.strftime('%Y-%m-%d') -# def traffic(environ, start_response): -# try: -# # reset_date = memcache.get(key="reset_date") -# reset_date = blobstore.get("reset_date") -# except: -# reset_date = None -# -# try: -# # traffic_sum = memcache.get(key="traffic") -# traffic_sum = blobstore.get("traffic") -# if not traffic_sum: -# traffic_sum = "0" -# except Exception as e: -# traffic_sum = "0" -# -# start_response('200 OK', [('Content-Type', 'text/plain')]) -# yield 'traffic:%s\r\n' % traffic_sum -# yield 'Reset date:%s\r\n' % reset_date -# yield 'Usage: %f %%\r\n' % int(int(traffic_sum) * 100 / allowed_traffic) -# -# tz = Pacific() -# sa_time = datetime.now(tz) -# pacific_time = sa_time.strftime('%Y-%m-%d %H:%M:%S') -# yield "American Pacific time:%s" % pacific_time -# -# raise StopIteration - - -# def reset(environ, start_response): -# try: -# # memcache.set(key="traffic", value="0") -# blobstore.set("traffic", "0") -# except: -# pass -# -# start_response('200 OK', [('Content-Type', 'text/plain')]) -# yield 'traffic reset finished.' -# raise StopIteration +def get_store_info(): + if not storage_client: + return {} + + try: + blob = bucket.blob(blob_name) + content = blob.download_as_string() + info = json.loads(content) + return info + except Exception as e: + logging.exception("get_store_info e:%r", e) + return {} + + +store_info = get_store_info() +store_save_time = time.time() +store_change_time = 0 +traffic = 0 +timer_th = None + + +def get_store(key, default_value=None): + global store_info + return store_info.get(key, default_value) + + +def set_store(key, value): + global store_info + store_info[key] = value + + +def save_store(): + global store_info, store_save_time, traffic + if not storage_client: + return + + store_info = get_store_info() + + try: + store_info["traffic"] = store_info.get("traffic", 0) + traffic + traffic = 0 + content = json.dumps(store_info) + blob = bucket.blob(blob_name) + blob.upload_from_string(content) + + store_save_time = time.time() + logging.info("save_store:%s", content) + except Exception as e: + logging.exception("save_store e:%r", e) + + +def timer_worker(): + global store_info, store_save_time, store_change_time, timer_th + + time.sleep(60 * 14) + if store_change_time > store_save_time and time.time() - store_save_time > 60: + logging.info("timer save store") + save_store() + timer_th = None + + +@app.route("/reset_traffic", methods=['GET']) +def reset_traffic(): + global traffic + traffic = 0 + set_store("traffic", 0) + save_store() + return 'Traffic reset finished.' def is_traffic_exceed(): - return False -# try: -# # reset_date = memcache.get(key="reset_date") -# reset_date = blobstore.get("reset_date") -# except: -# reset_date = None -# -# pacific_date = get_pacific_date() -# if reset_date != pacific_date: -# # memcache.set(key="reset_date", value=pacific_date) -# # memcache.set(key="traffic", value="0") -# blobstore.set("reset_date", pacific_date) -# blobstore.set("traffic", "0") -# return False -# -# try: -# # traffic_sum = int(memcache.get(key="traffic")) -# traffic_sum = int(blobstore.get("traffic")) -# except: -# traffic_sum = 0 -# -# if traffic_sum > allowed_traffic: -# return True -# else: -# return False + global traffic + reset_date = get_store("reset_date", None) + + pacific_date = get_pacific_date() + if reset_date != pacific_date: + set_store("reset_date", pacific_date) + traffic = 0 + set_store("traffic", 0) + save_store() + return False + + traffic_sum = get_store("traffic", 0) + traffic + + if traffic_sum > allowed_traffic: + return True + else: + return False def count_traffic(add_traffic): - pass -# try: -# # traffic_sum = int(memcache.get(key="traffic")) -# traffic_sum = int(blobstore.get("traffic")) -# except: -# traffic_sum = 0 -# -# try: -# v = str(traffic_sum + add_traffic) -# # memcache.set(key="traffic", value=v) -# blobstore.set("traffic", v) -# except Exception as e: -# logging.exception('memcache.set fail:%r', e) + global timer_th, traffic, store_change_time + traffic += add_traffic + store_change_time = time.time() + + if not timer_th: + timer_th = threading.Thread(target=timer_worker) + timer_th.start() @app.route("/") @@ -246,22 +289,157 @@ def root(): return out +@app.route("/info", methods=['GET']) +def info(): + global store_info, traffic + save_store() + + out_list = list() + out_list.append(f"store: 

{json.dumps(store_info, indent=2)}
") + + traffic_sum = int(store_info.get("traffic", 0)) + traffic + out_list.append('Usage: %f %%\r\n' % (traffic_sum * 100 / allowed_traffic)) + + tz = Pacific() + sa_time = datetime.now(tz) + pacific_time = sa_time.strftime('%Y-%m-%d %H:%M:%S') + out_list.append("American Pacific time:%s" % pacific_time) + + out_list.append(f"CPU num:{psutil.cpu_count()}") + out_list.append(f"CPU percent:{psutil.cpu_percent(interval=1)}") + mem = psutil.virtual_memory() + out_list.append(f"Mem total:{mem.total}") + out_list.append(f"Mem used:{mem.used}") + out_list.append(f"Mem available:{mem.available}") + out_list.append(f"Mem percent:{mem.percent}") + net = psutil.net_io_counters() + out_list.append(f"net sent:{net.bytes_sent}") + out_list.append(f"net recv:{net.bytes_recv}") + + env = json.dumps(dict(os.environ), indent=2) + out_list.append(f"
env: 
{env}
") + + return "
\r\n".join(out_list) + + @app.route("/_gh/", methods=['GET']) def check(): logging.debug("req headers:%s", request.headers) return "GoAgent works" -def req_by_requests(method, url, req_headers, req_body, timeout, verify, kwargs): - # maxsize = int(kwargs.get('maxsize', 0)) - # accept_encoding = headers.get('Accept-Encoding', '') +def req_by_requests(method, url, req_headers, req_body, kwargs): + timeout = int(kwargs.get('timeout', URLFETCH_TIMEOUT)) + verify = bool(int(kwargs.get('validate', 0))) errors = [] for i in range(int(kwargs.get('fetchmax', URLFETCH_MAX))): try: + # t0 = time.time() res = requests.request(method, url, headers=req_headers, data=req_body, timeout=timeout, verify=verify, stream=True, allow_redirects=False) + # t1 = time.time() + # logging.info(f"cost:{t1-t0} {method} {url} res:{res.status_code}") + break + except Exception as e: + logging.warning("request %s %s %s %s %s e:%r", method, url, req_headers, timeout, verify, e) + errors.append(str(e)) + if i == 0 and method == 'GET': + timeout *= 2 + else: + error_string = '
\n'.join(errors) + logging.info('%s %s error:%s', method, url, error_string) + return 502, {}, "502 Urlfetch Error: " + error_string + + res_code = res.status_code + res_headers = dict(res.headers) + content_length = int(res_headers.get("Content-Length", 0)) + + maxsize = int(kwargs.get('maxsize', URLFETCH_DEFLATE_MAXSIZE)) + if (method == "GET" and res_code == 200 and content_length and maxsize and content_length > maxsize and + "Range" not in req_headers and + res_headers.get('Accept-Ranges', '').lower() == 'bytes'): + + res_code = 206 + res_headers['Content-Range'] = 'bytes 0-%d/%d' % (maxsize - 1, content_length) + res_content = res.raw.read(maxsize) + logging.info("get %s data len:%d max:%d", url, content_length, maxsize) + else: + res_content = res.raw.read() + + if "Transfer-Encoding" in res_headers: + del res_headers["Transfer-Encoding"] + + res_headers["X-Head-Content-Length"] = res_headers["Content-Length"] = len(res_content) + return res_code, res_headers, res_content + + +def req_by_urlfetch(method, url, req_headers, req_body, kwargs): + timeout = int(kwargs.get('timeout', URLFETCH_TIMEOUT)) + maxsize = int(kwargs.get('maxsize', URLFETCH_DEFLATE_MAXSIZE)) + verify = bool(int(kwargs.get('validate', 0))) + + errors = [] + allow_truncated = False + for i in range(int(kwargs.get('fetchmax', URLFETCH_MAX))): + try: + t0 = time.time() + + res = urlfetch.fetch(url, req_body, method, req_headers, + allow_truncated=allow_truncated, + follow_redirects=False, + deadline=timeout, + validate_certificate=False) + # res = requests.request(method, url, headers=req_headers, data=req_body, timeout=timeout, verify=verify, + # stream=True, allow_redirects=False) + t1 = time.time() + logging.info(f"cost:{t1-t0} {method} {url} res:{res.status_code}") break + + except apiproxy_errors.OverQuotaError as e: + logging.info('%s %s OverQuotaError:%r', method, url, e) + return 510, {}, "510 Traffic exceed" + except urlfetch.SSLCertificateError as e: + errors.append('%r, should validate=0 ?' % e) + logging.warning('%r, timeout=%s', e, timeout) + + return 502, {}, "502 SSLCertificateError" + except urlfetch.DeadlineExceededError as e: + errors.append('%r, timeout=%s' % (e, timeout)) + logging.warning('DeadlineExceededError(timeout=%s, url=%r)', timeout, url) + time.sleep(1) + + allow_truncated = True + m = re.search(r'=\s*(\d+)-', req_headers.get('Range') or req_headers.get('range') or '') + if m is None: + req_headers['Range'] = 'bytes=0-%d' % (maxsize) + else: + req_headers.pop('Range', '') + req_headers.pop('range', '') + start = int(m.group(1)) + req_headers['Range'] = 'bytes=%s-%d' % (start, start + maxsize) + + timeout *= 2 + except urlfetch.DownloadError as e: + errors.append('%r, timeout=%s' % (e, timeout)) + logging.warning('DownloadError(timeout=%s, url=%r)', timeout, url) + time.sleep(1) + timeout *= 2 + except urlfetch.ResponseTooLargeError as e: + errors.append('%r, timeout=%s' % (e, timeout)) + response = e.response + logging.warning('ResponseTooLargeError(timeout=%s, url=%r) response(%r)', + timeout, url, response) + + m = re.search(r'=\s*(\d+)-', req_headers.get('Range') or req_headers.get('range') or '') + if m is None: + req_headers['Range'] = 'bytes=0-%d' % (maxsize) + else: + req_headers.pop('Range', '') + req_headers.pop('range', '') + start = int(m.group(1)) + req_headers['Range'] = 'bytes=%s-%d' % (start, start + (maxsize)) + timeout *= 2 except Exception as e: logging.warning("request %s %s %s %s %s e:%r", method, url, req_headers, timeout, verify, e) errors.append(str(e)) @@ -269,29 +447,39 @@ def req_by_requests(method, url, req_headers, req_body, timeout, verify, kwargs) timeout *= 2 else: error_string = '
\n'.join(errors) - logging.info('%s "%s %s" error:%s', request.remote_addr, method, url, error_string) + logging.warning('%s %s error:%s', method, url, error_string) return 502, {}, "502 Urlfetch Error: " + error_string + res_code = res.status_code res_headers = dict(res.headers) - res_content = res.raw.read() - # logging.debug(f'url={url} status_code={res.status_code} headers={res_headers} content={len(res_content)}') + content_length = int(res_headers.get("Content-Length", 0)) + + if (method == "GET" and res_code == 200 and content_length and maxsize and content_length > maxsize and + "Range" not in req_headers and res_headers.get('Accept-Ranges', '').lower() == 'bytes'): + + res_code = 206 + res_headers['Content-Range'] = 'bytes 0-%d/%d' % (maxsize - 1, content_length) + res_content = res.content[:maxsize] + logging.info("get %s data len:%d max:%d", url, content_length, maxsize) + else: + res_content = res.content if "Transfer-Encoding" in res_headers: del res_headers["Transfer-Encoding"] res_headers["X-Head-Content-Length"] = res_headers["Content-Length"] = len(res_content) - return res.status_code, res_headers, res_content + return res_code, res_headers, res_content @app.route("/_gh/", methods=['POST']) def proxy(): + if is_traffic_exceed(): + logging.info('Traffic exceed') + return "510 Traffic exceed", 510 + t0 = time.time() try: - method, url, req_headers, req_body, timeout, kwargs = unpack_request(request.data) - method = to_str(method) - url = to_str(url) - req_headers = to_str(req_headers) - kwargs = to_str(kwargs) + method, url, req_headers, req_body, kwargs = unpack_request(request.data) except Exception as e: logging.exception("unpack request:%r", e) return "500 Bad Request", 500 @@ -303,19 +491,14 @@ def proxy(): logging.info('wrong password') return "401 Wrong password", 401 - if is_traffic_exceed(): - logging.info('Traffic exceed') - return "510 Traffic exceed", 510 - netloc = urlparse(url).netloc if netloc.startswith(('127.0.0.', '::1', 'localhost')): return "GoAgent is Running", 400 - timeout = int(kwargs.get('timeout', URLFETCH_TIMEOUT)) - verify = bool(int(kwargs.get('validate', 0))) - - res_code, res_headers, res_body = req_by_requests(method, url, req_headers, req_body, timeout, verify, kwargs, ) - # res_code, res_headers, res_body = req_by_urlopen(method, url, req_headers, req_body, timeout, verify, kwargs, ) + if len(url) > MAX_URL_LENGTH or method not in gae_support_methods: + res_code, res_headers, res_body = req_by_requests(method, url, req_headers, req_body, kwargs) + else: + res_code, res_headers, res_body = req_by_urlfetch(method, url, req_headers, req_body, kwargs) res_data = pack_response(res_code, res_headers, b"", res_body) t1 = time.time() @@ -324,6 +507,7 @@ def proxy(): logging.info("cost:%f %s %s res_len:%d", cost, method, url, len(res_body)) count_traffic(len(request.data) + len(res_data)) + resp = flask.Response(res_data) resp.headers['Content-Type'] = 'image/gif' return resp @@ -331,10 +515,5 @@ def proxy(): if __name__ == "__main__": # This is used when running locally only. When deploying to Google App - # Engine, a webserver process such as Gunicorn will serve the app. This - # can be configured by adding an `entrypoint` to app.yaml. - # Flask's development server will automatically serve static files in - # the "static" directory. See: - # http://flask.pocoo.org/docs/1.0/quickstart/#static-files. Once deployed, - # App Engine itself will serve those files as configured in app.yaml. + # Engine, a webserver process such as Gunicorn will serve the app. app.run(host="127.0.0.1", port=8080, debug=True) diff --git a/code/default/gae_proxy/server/gae/requirements.txt b/code/default/gae_proxy/server/gae/requirements.txt index 74be077317..9665bcb701 100644 --- a/code/default/gae_proxy/server/gae/requirements.txt +++ b/code/default/gae_proxy/server/gae/requirements.txt @@ -1,4 +1,6 @@ Flask==3.0.0 requests gunicorn -gevent \ No newline at end of file +google-cloud-storage +appengine-python-standard +psutil \ No newline at end of file diff --git a/code/default/gae_proxy/tests/test_protocol.py b/code/default/gae_proxy/tests/test_protocol.py index a3a298d149..4bbcc4a17e 100644 --- a/code/default/gae_proxy/tests/test_protocol.py +++ b/code/default/gae_proxy/tests/test_protocol.py @@ -63,9 +63,13 @@ class Conf(object): config = Conf() -def pack_request(method, url, headers, body, timeout): - headers = dict(headers) - if isinstance(body, bytes) and body: +def pack_request(method, url, headers, body, kwargs): + method = utils.to_bytes(method) + url = utils.to_bytes(url) + headers = utils.to_bytes(headers) + body = utils.to_bytes(body) + kwargs = utils.to_bytes(kwargs) + if body: if len(body) < 10 * 1024 * 1024 and b'Content-Encoding' not in headers: # 可以压缩 zbody = deflate(body) @@ -73,14 +77,13 @@ def pack_request(method, url, headers, body, timeout): body = zbody headers[b'Content-Encoding'] = b'deflate' if len(body) > 10 * 1024 * 1024: - xlog.warn("body len:%d %s %s", len(body), method, url) + xlog.warn("body len:%d %s %s", len(body), utils.to_bytes(method), utils.to_bytes(url)) headers[b'Content-Length'] = utils.to_bytes(str(len(body))) # GAE don't allow set `Host` header if b'Host' in headers: del headers[b'Host'] - kwargs = {} # gae 用的参数 if config.GAE_PASSWORD: kwargs[b'password'] = config.GAE_PASSWORD @@ -91,7 +94,6 @@ def pack_request(method, url, headers, body, timeout): kwargs[b'maxsize'] = config.JS_MAXSIZE else: kwargs[b'maxsize'] = config.AUTORANGE_MAXSIZE - kwargs[b'timeout'] = str(timeout) # gae 用的参数　ｅｎｄ payload = b'%s %s HTTP/1.1\r\n' % (method, url) @@ -260,23 +262,19 @@ def test_pack_real_response(self): print(f"res_headers:{res_headers}") logging.debug(f"body:{body}") - def test_req_local(self): - method = b"GET" - url = b"http://www.github.com/" - # info = { - # "User-Agent": "python" - # } - # body = utils.to_bytes(json.dumps(info)) - body = b"" + def test_req_local(self, url="http://speedtest.ftp.otenet.gr/files/test10Mb.db"): + method = "GET" + body = "" headers = { # "Content-Length": str(len(body)), # "Content-Type": "application/json" "Accept-Encoding": "gzip, br" } - headers = utils.to_bytes(headers) - timeout = 30 + kwargs = { + "timeout": "30" + } - request_headers, payload = pack_request(method, url, headers, body, timeout) + request_headers, payload = pack_request(method, url, headers, body, kwargs) res = requests.post("http://localhost:8080/_gh/", data=payload) @@ -287,3 +285,7 @@ def test_req_local(self): logging.debug(f"res_headers:{json.dumps(res_headers, indent=2)}") logging.debug(f"body_len:{len(body)}") logging.debug(f"body_100:{body[:100]}") + + def test_local_req(self): + url = "http://github.com" + self.test_req_local(url) \ No newline at end of file