diff --git a/.gitignore b/.gitignore index f0efff6..5a3b40b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ dist/ *.egg-info *.egg .idea +*~ +*.pyc diff --git a/.travis.yml b/.travis.yml index 58a562a..4b228a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python python: - "2.7" + - "3.5" install: - python setup.py install before_script: diff --git a/AUTHORS b/AUTHORS index 05d3c80..32a873c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,6 +7,9 @@ Contributors: * yimiqisan (yimiqisan@gmail.com); * softlns (softliunaisen@gmai.com); +* Justin (justinli.ljt@gmail.com); * teknolog2000 (https://github.com/teknolog2000); * ChaosEternal (https://github.com/ChaosEternal); * SimplicityGuy (https://github.com/SimplicityGuy); +* Roy Williams (https://github.com/rowillia); +* Vinod Gupta (https://github.com/codervinod) diff --git a/setup.py b/setup.py index 1f33a0b..78d6cc9 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,11 @@ +from __future__ import absolute_import import re import ast from setuptools import setup _version_re = re.compile(r'__version__\s+=\s+(.*)') -with open('swagger_py_codegen/__init__.py', 'rb') as f: +with open('swagger_py_codegen/_version.py', 'rb') as f: version = str(ast.literal_eval(_version_re.search( f.read().decode('utf-8')).group(1))) @@ -23,11 +24,12 @@ 'swagger_py_codegen=swagger_py_codegen:generate' ] }, - install_requires=['PyYAML', 'click', 'jinja2', 'dpath'], + install_requires=['PyYAML', 'click', 'jinja2', 'dpath', 'six'], tests_require=['pytest'], classifiers=[ 'Development Status :: 3 - Alpha', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', ], ) diff --git a/swagger_py_codegen/__init__.py b/swagger_py_codegen/__init__.py index 532d63d..1afbf73 100644 --- a/swagger_py_codegen/__init__.py +++ b/swagger_py_codegen/__init__.py @@ -1,3 +1,4 @@ -from .command import generate +from __future__ import absolute_import -__version__ = '0.1.19' +from ._version import __version__ +from .command import generate diff --git a/swagger_py_codegen/__main__.py b/swagger_py_codegen/__main__.py index e432b33..c7ee60b 100644 --- a/swagger_py_codegen/__main__.py +++ b/swagger_py_codegen/__main__.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + from .command import generate generate() diff --git a/swagger_py_codegen/_version.py b/swagger_py_codegen/_version.py new file mode 100644 index 0000000..75a44a9 --- /dev/null +++ b/swagger_py_codegen/_version.py @@ -0,0 +1,4 @@ +"""Version information.""" + +# The following line *must* be the last in the module, exactly as formatted: +__version__ = "0.2.4" diff --git a/swagger_py_codegen/base.py b/swagger_py_codegen/base.py index d2b172d..1d31c94 100644 --- a/swagger_py_codegen/base.py +++ b/swagger_py_codegen/base.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import os from jinja2 import Environment, FileSystemLoader diff --git a/swagger_py_codegen/command.py b/swagger_py_codegen/command.py index 31f1767..693d950 100644 --- a/swagger_py_codegen/command.py +++ b/swagger_py_codegen/command.py @@ -1,14 +1,17 @@ +from __future__ import absolute_import import codecs try: import simplejson as json except ImportError: import json +from multiprocessing import Pool from os import makedirs from os.path import join, exists, dirname import yaml import click +from ._version import __version__ from .flask import FlaskGenerator from .parser import Swagger from .base import Template @@ -53,6 +56,13 @@ def _copy_ui_dir(ui_dest, ui_src): return status +def print_version(ctx, param, value): + if not value or ctx.resilient_parsing: + return + click.echo('current version: %s' % __version__) + ctx.exit() + + @click.command() @click.argument('destination', required=True) @click.option('-s', '--swagger', '--swagger-doc', @@ -69,12 +79,17 @@ def _copy_ui_dir(ui_dest, ui_src): @click.option('--ui', default=False, is_flag=True, help='Generate swagger ui.') +@click.option('-j', '--jobs', + default=4, help='Parallel jobs for processing.') +@click.option('--version', is_flag=True, callback=print_version, + expose_value=False, is_eager=True, + help='Show current version.') def generate(destination, swagger_doc, force=False, package=None, - template_dir=None, specification=False, ui=False): - + template_dir=None, specification=False, ui=False, jobs=4): + pool = Pool(processes=int(jobs)) package = package or destination.replace('-', '_') data = spec_load(swagger_doc) - swagger = Swagger(data) + swagger = Swagger(data, pool) generator = FlaskGenerator(swagger) generator.with_spec = specification generator.with_ui = ui diff --git a/swagger_py_codegen/flask.py b/swagger_py_codegen/flask.py index c162cac..5e67ac0 100644 --- a/swagger_py_codegen/flask.py +++ b/swagger_py_codegen/flask.py @@ -1,8 +1,10 @@ +from __future__ import absolute_import import re from collections import OrderedDict from .base import Code, CodeGenerator from .jsonschema import Schema, SchemaGenerator, build_default +import six SUPPORT_METHODS = ['get', 'post', 'put', 'delete', 'patch', 'options', 'head'] @@ -94,13 +96,21 @@ def _type(parameters): return url, params +if six.PY3: + def _remove_characters(text, deletechars): + return text.translate({ord(x): None for x in deletechars}) +else: + def _remove_characters(text, deletechars): + return text.translate(None, deletechars) def _path_to_endpoint(swagger_path): - return swagger_path.strip('/').replace('/', '_').replace('-', '_').translate(None, '{}') + return _remove_characters( + swagger_path.strip('/').replace('/', '_').replace('-', '_'), + '{}') def _path_to_resource_name(swagger_path): - return swagger_path.title().translate(None, '{}/_-') + return _remove_characters(swagger_path.title(), '{}/_-') def _location(swagger_location): @@ -130,18 +140,18 @@ def _dependence_callback(self, code): # use flask endpoint to replace default validator's key, # example: `('some_path_param', 'method')` validators = OrderedDict() - for k, v in schemas.data['validators'].iteritems(): - locations = {_location(loc): val for loc, val in v.iteritems()} + for k, v in six.iteritems(schemas.data['validators']): + locations = {_location(loc): val for loc, val in six.iteritems(v)} validators[(_path_to_endpoint(k[0]), k[1])] = locations # filters filters = OrderedDict() - for k, v in schemas.data['filters'].iteritems(): + for k, v in six.iteritems(schemas.data['filters']): filters[(_path_to_endpoint(k[0]), k[1])] = v # scopes scopes = OrderedDict() - for k, v in schemas.data['scopes'].iteritems(): + for k, v in six.iteritems(schemas.data['scopes']): scopes[(_path_to_endpoint(k[0]), k[1])] = v schemas.data['validators'] = validators @@ -169,11 +179,12 @@ def _process_data(self): methods[method] = {} validator = self.validators.get((endpoint, method.upper())) if validator: - methods[method]['requests'] = validator.keys() + methods[method]['requests'] = list(validator.keys()) - for status, res_data in data[method].get('responses', {}).iteritems(): + for status, res_data in six.iteritems(data[method].get('responses', {})): if isinstance(status, int) or status.isdigit(): - example = res_data.get('schema', {}).get('application/json') + example = res_data.get('examples', {}).get('application/json') + if not example: example = build_default(res_data.get('schema')) response = example, int(status), build_default(res_data.get('headers')) diff --git a/swagger_py_codegen/jsonschema.py b/swagger_py_codegen/jsonschema.py index 046410d..a72f132 100644 --- a/swagger_py_codegen/jsonschema.py +++ b/swagger_py_codegen/jsonschema.py @@ -1,8 +1,10 @@ +from __future__ import absolute_import from collections import OrderedDict from inspect import getsource from .base import Code, CodeGenerator from .parser import schema_var_name +import six class Schema(Code): @@ -74,7 +76,7 @@ def build_data(swagger): responses = data.get('responses') if responses: filter = {} - for status, res_data in responses.iteritems(): + for status, res_data in six.iteritems(responses): if isinstance(status, int) or status.isdigit(): filter[int(status)] = dict( headers=res_data.get('headers'), @@ -84,7 +86,7 @@ def build_data(swagger): # scopes for security in data.get('security', []): - scopes[(endpoint, method)] = security.values().pop() + scopes[(endpoint, method)] = list(security.values()).pop() break schemas = OrderedDict([(schema_var_name(path), swagger.get(path)) for path in swagger.definitions]) @@ -106,7 +108,7 @@ def _process(self): yield Schema(build_data(self.swagger)) -def merge_default(schema, value): +def merge_default(schema, value, get_first=True): # TODO: more types support type_defaults = { 'integer': 9573, @@ -116,7 +118,10 @@ def merge_default(schema, value): 'boolean': False } - return normalize(schema, value, type_defaults)[0] + results = normalize(schema, value, type_defaults) + if get_first: + return results[0] + return results def build_default(schema): @@ -125,6 +130,8 @@ def build_default(schema): def normalize(schema, data, required_defaults=None): + import six + if required_defaults is None: required_defaults = {} errors = [] @@ -147,8 +154,8 @@ def has(self, key): def keys(self): if isinstance(self.data, dict): - return self.data.keys() - return vars(self.data).keys() + return list(self.data.keys()) + return list(vars(self.data).keys()) def get_check(self, key, default=None): if isinstance(self.data, dict): @@ -169,13 +176,9 @@ def _normalize_dict(schema, data): if not isinstance(data, DataWrapper): data = DataWrapper(data) - for key, _schema in schema.get('properties', {}).iteritems(): + for key, _schema in six.iteritems(schema.get('properties', {})): # set default type_ = _schema.get('type', 'object') - if ('default' not in _schema - and key in schema.get('required', []) - and type_ in required_defaults): - _schema['default'] = required_defaults[type_] # get value value, has_key = data.get_check(key) @@ -184,8 +187,11 @@ def _normalize_dict(schema, data): elif 'default' in _schema: result[key] = _schema['default'] elif key in schema.get('required', []): - errors.append(dict(name='property_missing', - message='`%s` is required' % key)) + if type_ in required_defaults: + result[key] = required_defaults[type_] + else: + errors.append(dict(name='property_missing', + message='`%s` is required' % key)) for _schema in schema.get('allOf', []): rs_component = _normalize(_schema, data) diff --git a/swagger_py_codegen/parser.py b/swagger_py_codegen/parser.py index d2d2fb4..e4c7b9c 100644 --- a/swagger_py_codegen/parser.py +++ b/swagger_py_codegen/parser.py @@ -1,11 +1,14 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import string import copy import dpath.util - +import six +from six.moves import map +import sys def schema_var_name(path): - return ''.join(map(string.capitalize, path)) + return ''.join(map(str.capitalize, path)) class RefNode(dict): @@ -22,12 +25,16 @@ class Swagger(object): separator = '\0' - def __init__(self, data): + def __init__(self, data, pool=None): self.data = data self.origin_data = copy.deepcopy(data) self._definitions = [] self._references_sort() - self._process_ref() + self._get_cached = {} + if pool: + process_references(self, pool) + else: + self._process_ref() def _process_ref(self): @@ -51,7 +58,7 @@ def get_definition_refs(): ref = ref.lstrip('#/').split('/') ref = tuple(ref) - if schema in definition_refs.keys(): + if schema in list(definition_refs.keys()): definition_refs[schema].add(ref) else: definition_refs[schema] = set([ref]) @@ -61,13 +68,13 @@ def get_definition_refs(): definition_refs = get_definition_refs() while definition_refs: - ready = {definition for definition, refs in definition_refs.iteritems() if not refs} + ready = {definition for definition, refs in six.iteritems(definition_refs) if not refs} if not ready: msg = '$ref circular references found!\n' raise ValueError(msg) for definition in ready: del definition_refs[definition] - for refs in definition_refs.itervalues(): + for refs in six.itervalues(definition_refs): refs.difference_update(ready) self._definitions += ready @@ -76,8 +83,16 @@ def search(self, path): for p, d in dpath.util.search(self.data, list(path), True, self.separator): yield tuple(p.split(self.separator)), d + def pickle_search(self, path): + for p, d in dpath.util.search(self.data, list(path), True, + self.separator): + yield (self, tuple(p.split(self.separator)), d) + def get(self, path): - return dpath.util.get(self.data, list(path)) + key = ''.join(path) + if key not in self._get_cached: + self._get_cached[key] = dpath.util.get(self.data, list(path)) + return self._get_cached[key] def set(self, path, data): dpath.util.set(self.data, list(path), data) @@ -89,7 +104,7 @@ def definitions(self): @property def scopes_supported(self): for _, data in self.search(['securityDefinitions', '*', 'scopes']): - return data.keys() + return list(data.keys()) return [] @property @@ -99,3 +114,42 @@ def module_name(self): @property def base_path(self): return self.data.get('basePath', '/v1') + + +def process_input_func(data_to_process): + (swagger, path, ref) = data_to_process + sys.stdout.write('.') + sys.stdout.flush() + ref = ref.lstrip('#/').split('/') + ref = tuple(ref) + data = swagger.get(ref) + path = path[:-1] + return (path, RefNode(data, ref)) + + +def process_references(swagger, pool): + """ + Processed references in swagger data + :param swagger: + :return: + """ + data_set = pool.map(process_input_func, + swagger.pickle_search(['**', '$ref'])) + for path, node in data_set: + sys.stdout.write('.') + sys.stdout.flush() + next_ref = swagger.data + for pn in path[:-1]: + if isinstance(next_ref, list): + next_ref = next_ref[int(pn)] + elif isinstance(next_ref, dict): + if pn in next_ref: + next_ref = next_ref[pn] + elif int(pn) in next_ref: + next_ref = next_ref[int(pn)] + if isinstance(next_ref, dict): + idx = path[-1] + next_ref[idx] = node + elif isinstance(next_ref, list): + idx = int(path[-1]) + next_ref[idx] = node diff --git a/swagger_py_codegen/templates/flask/api.tpl b/swagger_py_codegen/templates/flask/api.tpl index c7c3389..9a079b6 100644 --- a/swagger_py_codegen/templates/flask/api.tpl +++ b/swagger_py_codegen/templates/flask/api.tpl @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import + import flask_restful as restful from ..validators import request_validate, response_filter diff --git a/swagger_py_codegen/templates/flask/app.tpl b/swagger_py_codegen/templates/flask/app.tpl index 5cf1788..e96a83c 100644 --- a/swagger_py_codegen/templates/flask/app.tpl +++ b/swagger_py_codegen/templates/flask/app.tpl @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import + from flask import Flask import {{ blueprint }} diff --git a/swagger_py_codegen/templates/flask/blueprint.tpl b/swagger_py_codegen/templates/flask/blueprint.tpl index 81d31a6..2ccb307 100644 --- a/swagger_py_codegen/templates/flask/blueprint.tpl +++ b/swagger_py_codegen/templates/flask/blueprint.tpl @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import + from flask import Blueprint import flask_restful as restful diff --git a/swagger_py_codegen/templates/flask/requirements.tpl b/swagger_py_codegen/templates/flask/requirements.tpl index 1816eaf..1786ac3 100644 --- a/swagger_py_codegen/templates/flask/requirements.tpl +++ b/swagger_py_codegen/templates/flask/requirements.tpl @@ -2,3 +2,4 @@ Flask Jinja2 Flask-RESTful jsonschema +six diff --git a/swagger_py_codegen/templates/flask/routers.tpl b/swagger_py_codegen/templates/flask/routers.tpl index 5e6a5de..3426541 100644 --- a/swagger_py_codegen/templates/flask/routers.tpl +++ b/swagger_py_codegen/templates/flask/routers.tpl @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- {% include '_do_not_change.tpl' %} +from __future__ import absolute_import {% for view in views -%} from .api.{{ view.endpoint }} import {{ view.name }} diff --git a/swagger_py_codegen/templates/flask/validators.tpl b/swagger_py_codegen/templates/flask/validators.tpl index 421f4fc..b60827a 100644 --- a/swagger_py_codegen/templates/flask/validators.tpl +++ b/swagger_py_codegen/templates/flask/validators.tpl @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- {% include '_do_not_change.tpl' %} +from __future__ import absolute_import from datetime import date from functools import wraps +import six + from werkzeug.datastructures import MultiDict, Headers from flask import request, g, current_app, json from flask_restful import abort @@ -28,20 +31,26 @@ class FlaskValidatorAdaptor(object): def __init__(self, schema): self.validator = Draft4Validator(schema) + def validate_number(self, type_, value): + try: + return type_(value) + except ValueError: + return value + def type_convert(self, obj): if obj is None: return None if isinstance(obj, (dict, list)) and not isinstance(obj, MultiDict): return obj if isinstance(obj, Headers): - obj = MultiDict(obj.iteritems()) + obj = MultiDict(six.iteritems(obj)) result = dict() convert_funs = { - 'integer': lambda v: int(v[0]), + 'integer': lambda v: self.validate_number(int, v[0]), 'boolean': lambda v: v[0].lower() not in ['n', 'no', 'false', '', '0'], 'null': lambda v: None, - 'number': lambda v: float(v[0]), + 'number': lambda v: self.validate_number(float, v[0]), 'string': lambda v: v[0] } @@ -49,7 +58,7 @@ class FlaskValidatorAdaptor(object): func = convert_funs.get(type_, lambda v: v[0]) return [func([i]) for i in v] - for k, values in obj.iterlists(): + for k, values in obj.lists(): prop = self.validator.schema['properties'].get(k, {}) type_ = prop.get('type') fun = convert_funs.get(type_, lambda v: v[0]) @@ -63,7 +72,7 @@ class FlaskValidatorAdaptor(object): def validate(self, value): value = self.type_convert(value) errors = list(e.message for e in self.validator.iter_errors(value)) - return merge_default(self.validator.schema, value), errors + return normalize(self.validator.schema, value)[0], errors def request_validate(view): @@ -80,7 +89,7 @@ def request_validate(view): if method == 'HEAD': method = 'GET' locations = validators.get((endpoint, method), {}) - for location, schema in locations.iteritems(): + for location, schema in six.iteritems(locations): value = getattr(request, location, MultiDict()) validator = FlaskValidatorAdaptor(schema) result, errors = validator.validate(value) @@ -115,7 +124,10 @@ def response_filter(view): resp, status, headers = unpack(resp) if len(filter) == 1: - status = filter.keys()[0] + if six.PY3: + status = list(filter.keys())[0] + else: + status = filter.keys()[0] schemas = filter.get(status) if not schemas: diff --git a/swagger_py_codegen/templates/flask/view.tpl b/swagger_py_codegen/templates/flask/view.tpl index 81eb7ca..7f48b8d 100644 --- a/swagger_py_codegen/templates/flask/view.tpl +++ b/swagger_py_codegen/templates/flask/view.tpl @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function + from flask import request, g from . import Resource @@ -7,11 +9,11 @@ from .. import schemas class {{ name }}(Resource): - {%- for method, ins in methods.iteritems() %} + {%- for method, ins in methods.items() %} def {{ method.lower() }}(self{{ params.__len__() and ', ' or '' }}{{ params | join(', ') }}): {%- for request in ins.requests %} - print g.{{ request }} + print(g.{{ request }}) {%- endfor %} {% if 'response' in ins -%} diff --git a/swagger_py_codegen/templates/jsonschema/schemas.tpl b/swagger_py_codegen/templates/jsonschema/schemas.tpl index 6f146ff..128fd1c 100644 --- a/swagger_py_codegen/templates/jsonschema/schemas.tpl +++ b/swagger_py_codegen/templates/jsonschema/schemas.tpl @@ -4,24 +4,24 @@ {% include '_do_not_change.tpl' %} -{% for name, value in schemas.iteritems() %} +{% for name, value in schemas.items() %} {{ name }} = {{ value }} {%- endfor %} validators = { -{%- for name, value in validators.iteritems() %} +{%- for name, value in validators.items() %} {{ name }}: {{ value }}, {%- endfor %} } filters = { -{%- for name, value in filters.iteritems() %} +{%- for name, value in filters.items() %} {{ name }}: {{ value }}, {%- endfor %} } scopes = { -{%- for name, value in scopes.iteritems() %} +{%- for name, value in scopes.items() %} {{ name }}: {{ value }}, {%- endfor %} } diff --git a/tests/test_flask.py b/tests/test_flask.py index a53d7e9..a694de4 100644 --- a/tests/test_flask.py +++ b/tests/test_flask.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + from swagger_py_codegen.parser import Swagger from swagger_py_codegen.flask import ( _swagger_to_flask_url, @@ -134,6 +136,7 @@ def test_process_data(): swagger = Swagger(data) generator = FlaskGenerator(swagger) schemas, routes, view1, view2 = list(generator.generate())[:4] + view1, view2 = sorted([view1, view2], key=lambda x: x.data['name']) assert ('posts_post_id', 'GET') in schemas.data['validators'] assert schemas.data['validators'][('posts_post_id', 'GET')]['args']['properties']['page']['type'] == 'integer' assert view1.data['url'] == '/posts/' diff --git a/tests/test_jsonschema.py b/tests/test_jsonschema.py index 1c6c05d..42a15fa 100644 --- a/tests/test_jsonschema.py +++ b/tests/test_jsonschema.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from swagger_py_codegen.parser import Swagger from swagger_py_codegen.jsonschema import build_data @@ -66,7 +67,7 @@ def test_schema_ref_01(): swagger = Swagger(data) data = build_data(swagger) assert len(data['schemas']) == 2 - assert data['schemas'].keys()[0] == 'DefinitionsUser' + assert list(data['schemas'].keys())[0] == 'DefinitionsUser' def test_validators(): @@ -290,7 +291,7 @@ def test_merge_default_01(): } } result = merge_default(schema, default) - assert 'roles' not in result.keys() + assert 'roles' not in list(result.keys()) def test_merge_default_02(): @@ -407,7 +408,7 @@ def age(self): del schema['items']['properties']['roles']['default'] users, errors = normalize(schema, [User()]) user = users.pop() - assert 'roles' not in user.keys() + assert 'roles' not in list(user.keys()) assert errors user = User() @@ -469,7 +470,7 @@ def test_normalize_02(): } } result, errors = normalize(schema, default) - assert 'roles' not in result.keys() + assert 'roles' not in list(result.keys()) assert errors @@ -511,7 +512,7 @@ def test_normalize_03(): result, errors = normalize(schema, default) assert errors == [] assert result['name'] == 'bob' - assert 'address' not in result.keys() + assert 'address' not in list(result.keys()) default = { 'id': 123, diff --git a/tests/test_parser.py b/tests/test_parser.py index da9e7e9..31ed0c9 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import pytest from swagger_py_codegen.parser import Swagger