diff --git a/.travis.yml b/.travis.yml index d21e1bb0..6a56ce80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,42 @@ python: - "2.6" - "2.7" -install: - - pip install flake8 - - pip install -e . +env: + global: + - PYTHONPATH=/tmp/gaesdk -before_script: + matrix: + - USE_EXTENSIONS=true # no dependencies + - USE_EXTENSIONS=false + - USE_EXTENSIONS=true DJANGO_VERSION=1.2.7 + - USE_EXTENSIONS=false DJANGO_VERSION=1.2.7 + - USE_EXTENSIONS=true DJANGO_VERSION=1.3.7 + - USE_EXTENSIONS=false DJANGO_VERSION=1.3.7 + - USE_EXTENSIONS=true DJANGO_VERSION=1.5.12 + - USE_EXTENSIONS=false DJANGO_VERSION=1.5.12 + - USE_EXTENSIONS=true SQLALCHEMY_VERSION=0.7.10 + - USE_EXTENSIONS=false SQLALCHEMY_VERSION=0.7.10 + - USE_EXTENSIONS=true SQLALCHEMY_VERSION=0.8.7 + - USE_EXTENSIONS=false SQLALCHEMY_VERSION=0.8.7 + - USE_EXTENSIONS=true SQLALCHEMY_VERSION=0.9.8 + - USE_EXTENSIONS=false SQLALCHEMY_VERSION=0.9.8 + - USE_EXTENSIONS=true TWISTED_VERSION=14.0.2 + - USE_EXTENSIONS=false TWISTED_VERSION=14.0.2 + - USE_EXTENSIONS=true GAESDK_VERSION=1.9.17 + - USE_EXTENSIONS=false GAESDK_VERSION=1.9.17 + +before_install: + - pip install flake8 - flake8 +install: + - pip install flake8 coverage coveralls + - pip install -e . + - ./install_optional_dependencies.sh + - ./maybe_install_cython.sh + script: - - python setup.py test - - pip install Cython - - python setup.py test + - coverage run --source=pyamf setup.py test + +after_success: + - coveralls diff --git a/MANIFEST.in b/MANIFEST.in index 975ac42c..e7ab7f3c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,10 +2,9 @@ graft doc graft pyamf/tests/imports prune doc/build prune doc/_build -include distribute_setup.py include setupinfo.py include *.txt global-exclude *.swf global-exclude *.pyc include cpyamf/*.pyx -include cpyamf/*.pxd \ No newline at end of file +include cpyamf/*.pxd diff --git a/README.txt b/README.rst similarity index 94% rename from README.txt rename to README.rst index b08af1c3..737a7a40 100644 --- a/README.txt +++ b/README.rst @@ -1,3 +1,6 @@ +.. image:: https://coveralls.io/repos/hydralabs/pyamf/badge.svg + :target: https://coveralls.io/r/hydralabs/pyamf + PyAMF_ provides Action Message Format (AMF_) support for Python_ that is compatible with the `Adobe Flash Player`_. It includes integration with Python web frameworks like Django_, Pylons_, Twisted_, SQLAlchemy_, diff --git a/cpyamf/amf3.pxd b/cpyamf/amf3.pxd index 8a1db9b6..a32ef444 100644 --- a/cpyamf/amf3.pxd +++ b/cpyamf/amf3.pxd @@ -20,7 +20,7 @@ cdef class ClassDefinition(object): cdef class Context(codec.Context): - cdef codec.IndexedCollection strings + cdef codec.ByteStringReferenceCollection strings cdef dict classes cdef dict class_ref cdef dict proxied_objects diff --git a/cpyamf/amf3.pyx b/cpyamf/amf3.pyx index 118e7d48..1b587c47 100644 --- a/cpyamf/amf3.pyx +++ b/cpyamf/amf3.pyx @@ -141,7 +141,7 @@ cdef class Context(codec.Context): """ def __cinit__(self): - self.strings = codec.IndexedCollection(use_hash=1) + self.strings = codec.ByteStringReferenceCollection() self.classes = {} self.class_ref = {} self.proxied_objects = {} diff --git a/cpyamf/codec.pxd b/cpyamf/codec.pxd index d779b8fb..3da87ec9 100644 --- a/cpyamf/codec.pxd +++ b/cpyamf/codec.pxd @@ -29,6 +29,17 @@ cdef class IndexedCollection(object): cpdef Py_ssize_t append(self, object obj) except -1 +cdef class ByteStringReferenceCollection(IndexedCollection): + """ + There have been rare hash collisions within a single AMF payload causing + corrupt payloads. + + Which strings cause collisions is dependent on the python runtime, each + platform might have a slightly different implementation which means that + testing is extremely difficult. + """ + + cdef class Context(object): """ C based version of ``pyamf.BaseContext`` diff --git a/cpyamf/codec.pyx b/cpyamf/codec.pyx index 8a1a27d3..7a522aee 100644 --- a/cpyamf/codec.pyx +++ b/cpyamf/codec.pyx @@ -120,7 +120,7 @@ cdef class IndexedCollection(object): return self.data[ref] - cdef inline object _ref(self, object obj): + cdef object _ref(self, object obj): if self.use_hash: return hash(obj) @@ -198,6 +198,25 @@ cdef class IndexedCollection(object): return n +cdef class ByteStringReferenceCollection(IndexedCollection): + """ + There have been rare hash collisions within a single AMF payload causing + corrupt payloads. + + Which strings cause collisions is dependent on the python runtime, each + platform might have a slightly different implementation which means that + testing is extremely difficult. + """ + + cdef object _ref(self, object obj): + return obj + + def __copy__(self): + cdef ByteStringReferenceCollection n = ByteStringReferenceCollection() + + return n + + cdef class Context(object): """ I hold the AMF context for en/decoding streams. diff --git a/distribute_setup.py b/distribute_setup.py deleted file mode 100644 index 1767bd66..00000000 --- a/distribute_setup.py +++ /dev/null @@ -1,482 +0,0 @@ -#!python -"""Bootstrap distribute installation - -If you want to use setuptools in your package's setup.py, just include this -file in the same directory with it, and add this to the top of your setup.py:: - - from distribute_setup import use_setuptools - use_setuptools() - -If you want to require a specific version of setuptools, set a download -mirror, or use an alternate download directory, you can do so by supplying -the appropriate options to ``use_setuptools()``. - -This file can also be run as a script to install or upgrade setuptools. -""" -import os -import sys -import time -import fnmatch -import tempfile -import tarfile -from distutils import log - -try: - from site import USER_SITE -except ImportError: - USER_SITE = None - -try: - import subprocess - - def _python_cmd(*args): - args = (sys.executable,) + args - return subprocess.call(args) == 0 - -except ImportError: - # will be used for python 2.3 - def _python_cmd(*args): - args = (sys.executable,) + args - # quoting arguments if windows - if sys.platform == 'win32': - def quote(arg): - if ' ' in arg: - return '"%s"' % arg - return arg - args = [quote(arg) for arg in args] - return os.spawnl(os.P_WAIT, sys.executable, *args) == 0 - -DEFAULT_VERSION = "0.6.14" -DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/" -SETUPTOOLS_FAKED_VERSION = "0.6c11" - -SETUPTOOLS_PKG_INFO = """\ -Metadata-Version: 1.0 -Name: setuptools -Version: %s -Summary: xxxx -Home-page: xxx -Author: xxx -Author-email: xxx -License: xxx -Description: xxx -""" % SETUPTOOLS_FAKED_VERSION - - -def _install(tarball): - # extracting the tarball - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - tar = tarfile.open(tarball) - _extractall(tar) - tar.close() - - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) - - # installing - log.warn('Installing Distribute') - if not _python_cmd('setup.py', 'install'): - log.warn('Something went wrong during the installation.') - log.warn('See the error message above.') - finally: - os.chdir(old_wd) - - -def _build_egg(egg, tarball, to_dir): - # extracting the tarball - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - tar = tarfile.open(tarball) - _extractall(tar) - tar.close() - - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) - - # building an egg - log.warn('Building a Distribute egg in %s', to_dir) - _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) - - finally: - os.chdir(old_wd) - # returning the result - log.warn(egg) - if not os.path.exists(egg): - raise IOError('Could not build the egg.') - - -def _do_download(version, download_base, to_dir, download_delay): - egg = os.path.join(to_dir, 'distribute-%s-py%d.%d.egg' - % (version, sys.version_info[0], sys.version_info[1])) - if not os.path.exists(egg): - tarball = download_setuptools(version, download_base, - to_dir, download_delay) - _build_egg(egg, tarball, to_dir) - sys.path.insert(0, egg) - import setuptools - setuptools.bootstrap_install_from = egg - - -def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, download_delay=15, no_fake=True): - # making sure we use the absolute path - to_dir = os.path.abspath(to_dir) - was_imported = 'pkg_resources' in sys.modules or \ - 'setuptools' in sys.modules - try: - try: - import pkg_resources - if not hasattr(pkg_resources, '_distribute'): - if not no_fake: - _fake_setuptools() - raise ImportError - except ImportError: - return _do_download(version, download_base, to_dir, download_delay) - try: - pkg_resources.require("distribute>="+version) - return - except pkg_resources.VersionConflict: - e = sys.exc_info()[1] - if was_imported: - sys.stderr.write( - "The required version of distribute (>=%s) is not available,\n" - "and can't be installed while this script is running. Please\n" - "install a more recent version first, using\n" - "'easy_install -U distribute'." - "\n\n(Currently using %r)\n" % (version, e.args[0])) - sys.exit(2) - else: - del pkg_resources, sys.modules['pkg_resources'] # reload ok - return _do_download(version, download_base, to_dir, - download_delay) - except pkg_resources.DistributionNotFound: - return _do_download(version, download_base, to_dir, - download_delay) - finally: - if not no_fake: - _create_fake_setuptools_pkg_info(to_dir) - -def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, delay=15): - """Download distribute from a specified location and return its filename - - `version` should be a valid distribute version number that is available - as an egg for download under the `download_base` URL (which should end - with a '/'). `to_dir` is the directory where the egg will be downloaded. - `delay` is the number of seconds to pause before an actual download - attempt. - """ - # making sure we use the absolute path - to_dir = os.path.abspath(to_dir) - from six.moves.urllib.request import urlopen - tgz_name = "distribute-%s.tar.gz" % version - url = download_base + tgz_name - saveto = os.path.join(to_dir, tgz_name) - src = dst = None - if not os.path.exists(saveto): # Avoid repeated downloads - try: - log.warn("Downloading %s", url) - src = urlopen(url) - # Read/write all in one block, so we don't create a corrupt file - # if the download is interrupted. - data = src.read() - dst = open(saveto, "wb") - dst.write(data) - finally: - if src: - src.close() - if dst: - dst.close() - return os.path.realpath(saveto) - -def _no_sandbox(function): - def __no_sandbox(*args, **kw): - try: - from setuptools.sandbox import DirectorySandbox - if not hasattr(DirectorySandbox, '_old'): - def violation(*args): - pass - DirectorySandbox._old = DirectorySandbox._violation - DirectorySandbox._violation = violation - patched = True - else: - patched = False - except ImportError: - patched = False - - try: - return function(*args, **kw) - finally: - if patched: - DirectorySandbox._violation = DirectorySandbox._old - del DirectorySandbox._old - - return __no_sandbox - -def _patch_file(path, content): - """Will backup the file then patch it""" - existing_content = open(path).read() - if existing_content == content: - # already patched - log.warn('Already patched.') - return False - log.warn('Patching...') - _rename_path(path) - f = open(path, 'w') - try: - f.write(content) - finally: - f.close() - return True - -_patch_file = _no_sandbox(_patch_file) - -def _same_content(path, content): - return open(path).read() == content - -def _rename_path(path): - new_name = path + '.OLD.%s' % time.time() - log.warn('Renaming %s into %s', path, new_name) - os.rename(path, new_name) - return new_name - -def _remove_flat_installation(placeholder): - if not os.path.isdir(placeholder): - log.warn('Unkown installation at %s', placeholder) - return False - found = False - for file in os.listdir(placeholder): - if fnmatch.fnmatch(file, 'setuptools*.egg-info'): - found = True - break - if not found: - log.warn('Could not locate setuptools*.egg-info') - return - - log.warn('Removing elements out of the way...') - pkg_info = os.path.join(placeholder, file) - if os.path.isdir(pkg_info): - patched = _patch_egg_dir(pkg_info) - else: - patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO) - - if not patched: - log.warn('%s already patched.', pkg_info) - return False - # now let's move the files out of the way - for element in ('setuptools', 'pkg_resources.py', 'site.py'): - element = os.path.join(placeholder, element) - if os.path.exists(element): - _rename_path(element) - else: - log.warn('Could not find the %s element of the ' - 'Setuptools distribution', element) - return True - -_remove_flat_installation = _no_sandbox(_remove_flat_installation) - -def _after_install(dist): - log.warn('After install bootstrap.') - placeholder = dist.get_command_obj('install').install_purelib - _create_fake_setuptools_pkg_info(placeholder) - -def _create_fake_setuptools_pkg_info(placeholder): - if not placeholder or not os.path.exists(placeholder): - log.warn('Could not find the install location') - return - pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1]) - setuptools_file = 'setuptools-%s-py%s.egg-info' % \ - (SETUPTOOLS_FAKED_VERSION, pyver) - pkg_info = os.path.join(placeholder, setuptools_file) - if os.path.exists(pkg_info): - log.warn('%s already exists', pkg_info) - return - - log.warn('Creating %s', pkg_info) - f = open(pkg_info, 'w') - try: - f.write(SETUPTOOLS_PKG_INFO) - finally: - f.close() - - pth_file = os.path.join(placeholder, 'setuptools.pth') - log.warn('Creating %s', pth_file) - f = open(pth_file, 'w') - try: - f.write(os.path.join(os.curdir, setuptools_file)) - finally: - f.close() - -_create_fake_setuptools_pkg_info = _no_sandbox(_create_fake_setuptools_pkg_info) - -def _patch_egg_dir(path): - # let's check if it's already patched - pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') - if os.path.exists(pkg_info): - if _same_content(pkg_info, SETUPTOOLS_PKG_INFO): - log.warn('%s already patched.', pkg_info) - return False - _rename_path(path) - os.mkdir(path) - os.mkdir(os.path.join(path, 'EGG-INFO')) - pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') - f = open(pkg_info, 'w') - try: - f.write(SETUPTOOLS_PKG_INFO) - finally: - f.close() - return True - -_patch_egg_dir = _no_sandbox(_patch_egg_dir) - -def _before_install(): - log.warn('Before install bootstrap.') - _fake_setuptools() - - -def _under_prefix(location): - if 'install' not in sys.argv: - return True - args = sys.argv[sys.argv.index('install')+1:] - for index, arg in enumerate(args): - for option in ('--root', '--prefix'): - if arg.startswith('%s=' % option): - top_dir = arg.split('root=')[-1] - return location.startswith(top_dir) - elif arg == option: - if len(args) > index: - top_dir = args[index+1] - return location.startswith(top_dir) - if arg == '--user' and USER_SITE is not None: - return location.startswith(USER_SITE) - return True - - -def _fake_setuptools(): - log.warn('Scanning installed packages') - try: - import pkg_resources - except ImportError: - # we're cool - log.warn('Setuptools or Distribute does not seem to be installed.') - return - ws = pkg_resources.working_set - try: - setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools', - replacement=False)) - except TypeError: - # old distribute API - setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools')) - - if setuptools_dist is None: - log.warn('No setuptools distribution found') - return - # detecting if it was already faked - setuptools_location = setuptools_dist.location - log.warn('Setuptools installation detected at %s', setuptools_location) - - # if --root or --preix was provided, and if - # setuptools is not located in them, we don't patch it - if not _under_prefix(setuptools_location): - log.warn('Not patching, --root or --prefix is installing Distribute' - ' in another location') - return - - # let's see if its an egg - if not setuptools_location.endswith('.egg'): - log.warn('Non-egg installation') - res = _remove_flat_installation(setuptools_location) - if not res: - return - else: - log.warn('Egg installation') - pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO') - if (os.path.exists(pkg_info) and - _same_content(pkg_info, SETUPTOOLS_PKG_INFO)): - log.warn('Already patched.') - return - log.warn('Patching...') - # let's create a fake egg replacing setuptools one - res = _patch_egg_dir(setuptools_location) - if not res: - return - log.warn('Patched done.') - _relaunch() - - -def _relaunch(): - log.warn('Relaunching...') - # we have to relaunch the process - # pip marker to avoid a relaunch bug - if sys.argv[:3] == ['-c', 'install', '--single-version-externally-managed']: - sys.argv[0] = 'setup.py' - args = [sys.executable] + sys.argv - sys.exit(subprocess.call(args)) - - -def _extractall(self, path=".", members=None): - """Extract all members from the archive to the current working - directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). - """ - import copy - import operator - from tarfile import ExtractError - directories = [] - - if members is None: - members = self - - for tarinfo in members: - if tarinfo.isdir(): - # Extract directories with a safe mode. - directories.append(tarinfo) - tarinfo = copy.copy(tarinfo) - tarinfo.mode = 448 # decimal for oct 0700 - self.extract(tarinfo, path) - - # Reverse sort directories. - if sys.version_info < (2, 4): - def sorter(dir1, dir2): - return cmp(dir1.name, dir2.name) - directories.sort(sorter) - directories.reverse() - else: - directories.sort(key=operator.attrgetter('name'), reverse=True) - - # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) - try: - self.chown(tarinfo, dirpath) - self.utime(tarinfo, dirpath) - self.chmod(tarinfo, dirpath) - except ExtractError: - e = sys.exc_info()[1] - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) - - -def main(argv, version=DEFAULT_VERSION): - """Install or upgrade setuptools and EasyInstall""" - tarball = download_setuptools() - _install(tarball) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/doc/conf.py b/doc/conf.py index 0c89cf00..a548aff8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -17,6 +17,7 @@ from docutils.core import publish_parts + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute. @@ -24,6 +25,12 @@ sys.path.append(os.path.abspath('.')) sys.path.append(os.path.abspath('html')) + +# When ReaTheDocs.org builds your project, it sets the READTHEDOCS environment +# variable to the string True. +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + + def rst2html(input, output): """ Create html file from rst file. @@ -57,8 +64,9 @@ def rst2html(input, output): extensions = ['sphinx.ext.intersphinx', 'sphinx.ext.extlinks', 'sphinxcontrib.epydoc'] -# Paths that contain additional templates, relative to this directory. -templates_path = ['html'] +if on_rtd == False: + # Paths that contain additional templates, relative to this directory. + templates_path = ['html'] # The suffix of source filenames. source_suffix = '.rst' @@ -74,8 +82,12 @@ def rst2html(input, output): project = 'PyAMF' url = 'http://pyamf.org' description = 'AMF for Python' -copyright = "Copyright © 2007-%s The %s Project. All rights reserved." % ( - time.strftime('%Y'), url, project) + +if on_rtd == False: + copyright = "Copyright © 2007-%s The %s Project. All rights reserved." % ( + time.strftime('%Y'), url, project) +else: + copyright = "2007-%s The %s Project" % (time.strftime('%Y'), project) # We look for the __init__.py file in the current PyAMF source tree # and replace the values accordingly. @@ -122,20 +134,30 @@ def rst2html(input, output): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# -# Note: you can download the 'beam' theme from: -# http://github.com/collab-project/sphinx-themes -# and place it in a 'themes' directory relative to this config file. -html_theme = 'beam' +if on_rtd: + # default theme for readthedocs.org + html_theme = 'default' +else: + # Note: you can download the 'beam' theme from: + # http://github.com/collab-project/sphinx-themes + # and place it in a 'themes' directory relative to this config file. + html_theme = 'beam' + + # Custom themes here, relative to this directory. + html_theme_path = ['themes'] + + # Additional templates that should be rendered to pages, maps page names to + # template names. + html_additional_pages = { + 'index': 'defindex.html', + 'tutorials/index': 'tutorials.html', + } # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} -# Add any paths that contain custom themes here, relative to this directory. -html_theme_path = ['themes'] - # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". html_title = '%s - %s' % (project, description) @@ -156,13 +178,6 @@ def rst2html(input, output): # typographically correct entities. #html_use_smartypants = True -# Additional templates that should be rendered to pages, maps page names to -# template names. -html_additional_pages = { - 'index': 'defindex.html', - 'tutorials/index': 'tutorials.html', -} - # If false, no module index is generated. html_use_modindex = True diff --git a/doc/install.rst b/doc/install.rst index 06c3b8f4..ded75128 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -4,31 +4,15 @@ .. contents:: -PyAMF requires Python_ 2.4 or newer. Python 3.0 isn't supported yet_. +PyAMF requires Python_ 2.4 or newer. Python 3.0 isn't supported. Easy Installation ================= -If you have setuptools_ or the `easy_install`_ tool already installed, -simply type the following on the command-line to install PyAMF:: +Pretty simple:: - easy_install pyamf - -`Note: you might need root permissions or equivalent for these steps.` - -If you don't have `setuptools` or `easy_install`, first download -distribute_setup.py_ and run:: - - python distribute_setup.py - -After `easy_install` is installed, run `easy_install pyamf` again. If -you run into problems, try the manual installation instructions below. - -To upgrade your existing PyAMF installation to the latest version -use:: - - easy_install -U pyamf + pip install pyamf Manual Installation @@ -193,10 +177,8 @@ folder. .. _Python: http://www.python.org -.. _yet: http://dev.pyamf.org/milestone/0.7 .. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools .. _easy_install: http://peak.telecommunity.com/DevCenter/EasyInstall#installing-easy-install -.. _distribute_setup.py: http://github.com/hydralabs/pyamf/blob/master/distribute_setup.py .. _Epydoc: http://epydoc.sourceforge.net .. _ElementTree: http://effbot.org/zone/element-index.htm .. _lxml: http://lxml.de @@ -214,6 +196,6 @@ folder. .. _Trial: http://twistedmatrix.com/trac/wiki/TwistedTrial .. _Cython: http://cython.org .. _Sphinx: http://sphinx.pocoo.org -.. _website: http://pyamf.org +.. _website: https://github.com/hydralabs/pyamf .. _Installing Python Modules: http://docs.python.org/install/index.html .. _sphinxcontrib.epydoc: http://packages.python.org/sphinxcontrib-epydoc diff --git a/install_optional_dependencies.sh b/install_optional_dependencies.sh new file mode 100755 index 00000000..3f88c9d0 --- /dev/null +++ b/install_optional_dependencies.sh @@ -0,0 +1,43 @@ +#!/bin/bash -e + +# used by Travis to install optional dependencies +# see .travis.yml:env for envrionment variables + +function install_django { + if [ -z "${DJANGO_VERSION}" ]; then + return 0 + fi + + pip install Django==${DJANGO_VERSION} +} + +function install_sqlalchemy { + if [ -z "${SQLALCHEMY_VERSION}" ]; then + return 0 + fi + + pip install SQLAlchemy==${SQLALCHEMY_VERSION} +} + +function install_twisted { + if [ -z "${TWISTED_VERSION}" ]; then + return 0 + fi + + pip install Twisted==${TWISTED_VERSION} +} + +function install_gae_sdk { + if [ -z "${GAESDK_VERSION}" ]; then + return 0 + fi + + wget https://storage.googleapis.com/appengine-sdks/featured/google_appengine_${GAESDK_VERSION}.zip -nv + mkdir -p /tmp/gaesdk + unzip -q google_appengine_${GAESDK_VERSION}.zip -d /tmp/gaesdk +} + +install_django +install_sqlalchemy +install_twisted +install_gae_sdk diff --git a/maybe_install_cython.sh b/maybe_install_cython.sh new file mode 100755 index 00000000..01437ef6 --- /dev/null +++ b/maybe_install_cython.sh @@ -0,0 +1,9 @@ +#!/bin/bash -e + +# used by travis to determine whether to install Cython (and thereby compile +# the PyAMF extensions) + + +if [ "${USE_EXTENSIONS}" == "true" ]; then + pip install Cython +fi diff --git a/pyamf/__init__.py b/pyamf/__init__.py index 40c8b187..738c5b6d 100644 --- a/pyamf/__init__.py +++ b/pyamf/__init__.py @@ -238,7 +238,7 @@ def getCustomProperties(self): def getEncodableAttributes(self, obj, **kwargs): attrs = ClassAlias.getEncodableAttributes(self, obj, **kwargs) - attrs['message'] = str(obj) + attrs['message'] = unicode(obj) attrs['name'] = obj.__class__.__name__ return attrs diff --git a/pyamf/adapters/_django_utils_translation.py b/pyamf/adapters/_django_utils_translation.py index b198c96c..5fbd60a0 100644 --- a/pyamf/adapters/_django_utils_translation.py +++ b/pyamf/adapters/_django_utils_translation.py @@ -14,8 +14,12 @@ def convert_lazy(l, encoder=None): - if l.__class__._delegate_unicode: - return u'1' + try: + if l.__class__._delegate_text: + return u'l' + except AttributeError: + if l.__class__._delegate_unicode: + return u'l' if l.__class__._delegate_str: return b'1' diff --git a/pyamf/alias.py b/pyamf/alias.py index 3ce2cbb0..fff29f76 100644 --- a/pyamf/alias.py +++ b/pyamf/alias.py @@ -501,25 +501,25 @@ def getDecodableAttributes(self, obj, attrs, codec=None): attrs[k] = context.getObjectForProxy(v) + if changed: + # apply all filters before synonyms + a = {} + + [a.__setitem__(p, attrs[p]) for p in props] + attrs = a + if self.synonym_attrs: missing = object() for k, v in iteritems(self.synonym_attrs): - value = attrs.pop(k, missing) + value = attrs.pop(v, missing) if value is missing: continue - attrs[v] = value + attrs[k] = value - if not changed: - return attrs - - a = {} - - [a.__setitem__(p, attrs[p]) for p in props] - - return a + return attrs def applyAttributes(self, obj, attrs, codec=None): """ diff --git a/pyamf/amf3.py b/pyamf/amf3.py index 338fb9aa..c5761ed5 100644 --- a/pyamf/amf3.py +++ b/pyamf/amf3.py @@ -87,7 +87,7 @@ #: table. #: @see: U{OSFlash documentation (external) #: } +#: #x07_-_xml_legacy_flash.xml.xmldocument_class>} TYPE_XML = b'\x07' #: In AMF 3 an ActionScript Date is serialized as the number of #: milliseconds elapsed since the epoch of midnight, 1st Jan 1970 in the @@ -156,6 +156,8 @@ class ObjectEncoding: class DataOutput(object): """ I am a C{StringIO} type object containing byte data from the AMF stream. + ActionScript 3.0 introduced the C{flash.utils.ByteArray} class to support + the manipulation of raw data in the form of an Array of bytes. I provide a set of methods for writing binary data with ActionScript 3.0. This class is the I/O counterpart to the L{DataInput} class, which reads @@ -602,13 +604,13 @@ class Context(codec.Context): I hold the AMF3 context for en/decoding streams. @ivar strings: A list of string references. - @type strings: C{list} + @type strings: L{codec.ByteStringReferenceCollection} @ivar classes: A list of L{ClassDefinition}. @type classes: C{list} """ def __init__(self): - self.strings = codec.IndexedCollection(use_hash=True) + self.strings = codec.ByteStringReferenceCollection() self.classes = {} self.class_ref = {} @@ -1108,14 +1110,17 @@ def readByteArray(self): return self.context.getObject(ref >> 1) buffer = self.stream.read(ref >> 1) + compressed = False if buffer[0:2] == ByteArray._zlib_header: try: buffer = zlib.decompress(buffer) + compressed = True except zlib.error: pass obj = ByteArray(buffer) + obj.compressed = compressed self.context.addObject(obj) diff --git a/pyamf/codec.py b/pyamf/codec.py index b9551b32..e91164e3 100644 --- a/pyamf/codec.py +++ b/pyamf/codec.py @@ -114,6 +114,30 @@ def __repr__(self): id(self)) +class ByteStringReferenceCollection(IndexedCollection): + """ + There have been rare hash collisions within a single AMF payload causing + corrupt payloads. + + Which strings cause collisions is dependent on the python runtime, each + platform might have a slightly different implementation which means that + testing is extremely difficult. + """ + + def __init__(self, *args, **kwargs): + super(ByteStringReferenceCollection, self).__init__(use_hash=False) + + def getReferenceTo(self, byte_string): + return self.dict.get(byte_string, -1) + + def append(self, byte_string): + self.list.append(byte_string) + idx = len(self.list) - 1 + self.dict[byte_string] = idx + + return idx + + class Context(object): """ The base context for all AMF [de|en]coding. diff --git a/pyamf/remoting/amf3.py b/pyamf/remoting/amf3.py index 1e94fa30..32dd4e9e 100644 --- a/pyamf/remoting/amf3.py +++ b/pyamf/remoting/amf3.py @@ -65,11 +65,14 @@ def generate_error(request, cls, e, tb, include_traceback=False): code = cls.__name__ details = None - rootCause = None + rootCause = e if include_traceback: - details = traceback.format_exception(cls, e, tb) - rootCause = e + buffer = pyamf.util.BufferedByteStream() + + traceback.print_exception(cls, e, tb, file=buffer) + + details = buffer.getvalue() faultDetail = None faultString = None diff --git a/pyamf/remoting/gateway/django.py b/pyamf/remoting/gateway/django.py index c083926b..94c9da16 100644 --- a/pyamf/remoting/gateway/django.py +++ b/pyamf/remoting/gateway/django.py @@ -94,17 +94,15 @@ def __call__(self, http_request): stream = None timezone_offset = self._get_timezone_offset() + try: + body = http_request.body + except AttributeError: + body = http_request.raw_post_data + # Decode the request try: request = remoting.decode( - http_request.raw_post_data, - strict=self.strict, - logger=self.logger, - timezone_offset=timezone_offset - ) - except AttributeError: # fix to make work with Django 1.6 - request = remoting.decode( - http_request.body, + body, strict=self.strict, logger=self.logger, timezone_offset=timezone_offset diff --git a/pyamf/remoting/gateway/google.py b/pyamf/remoting/gateway/google.py index 12513bd4..be999a79 100644 --- a/pyamf/remoting/gateway/google.py +++ b/pyamf/remoting/gateway/google.py @@ -69,7 +69,7 @@ def get(self): ) def post(self): - body = self.request.body_file.read() + body = self.request.body stream = None timezone_offset = self._get_timezone_offset() diff --git a/pyamf/tests/adapters/django_app/settings.py b/pyamf/tests/adapters/django_app/settings.py index b1e1054e..904a9f7a 100644 --- a/pyamf/tests/adapters/django_app/settings.py +++ b/pyamf/tests/adapters/django_app/settings.py @@ -3,7 +3,19 @@ # The simplest Django settings possible +# support for Django < 1.5 DATABASE_ENGINE = 'sqlite3' DATABASE_NAME = ':memory:' +# support for Django >= 1.5 +SECRET_KEY = 'unittest' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.' + DATABASE_ENGINE, + 'NAME': DATABASE_NAME, + } +} + + INSTALLED_APPS = ('adapters',) diff --git a/pyamf/tests/adapters/test_django.py b/pyamf/tests/adapters/test_django.py index 63069ba8..87116626 100644 --- a/pyamf/tests/adapters/test_django.py +++ b/pyamf/tests/adapters/test_django.py @@ -25,12 +25,8 @@ if django and django.VERSION < (1, 0): django = None -try: - reload(settings) -except NameError: - from pyamf.tests.adapters.django_app import settings - +settings = None context = None #: django modules/functions used once bootstrapped @@ -50,12 +46,14 @@ def init_django(): """ Bootstrap Django and initialise this module """ - global django, management, create_test_db, destroy_test_db + global django, management, create_test_db, destroy_test_db, settings global setup_test_environment, teardown_test_environment if not django: return + from pyamf.tests.adapters.django_app import settings + from django.core import management project_dir = management.setup_environ(settings) @@ -103,16 +101,16 @@ def tearDownModule(): if teardown_test_environment: teardown_test_environment() - sys.path = context['sys.path'] - util.replace_dict(context['sys.modules'], sys.modules) - util.replace_dict(context['os.environ'], os.environ) - if destroy_test_db: destroy_test_db(settings.DATABASE_NAME, verbosity=0) if storage: rmtree(storage.location, ignore_errors=True) + sys.path = context['sys.path'] + util.replace_dict(context['sys.modules'], sys.modules) + util.replace_dict(context['os.environ'], os.environ) + class BaseTestCase(unittest.TestCase): diff --git a/pyamf/tests/adapters/test_google.py b/pyamf/tests/adapters/test_google.py index 22e76e1d..73740ec3 100644 --- a/pyamf/tests/adapters/test_google.py +++ b/pyamf/tests/adapters/test_google.py @@ -7,25 +7,28 @@ @since: 0.3.1 """ -import unittest import datetime import struct -import os import pyamf from pyamf import amf3 from pyamf.tests import util + try: - from google.appengine.ext import db + import dev_appserver + + dev_appserver.fix_sys_path() except ImportError: - db = None + dev_appserver = None +db = None blobstore = None polymodel = None adapter_db = None adapter_blobstore = None +testbed = None test_models = None @@ -36,17 +39,17 @@ def setUpModule(): """ """ global db, blobstore, polymodel, adapter_blobstore, adapter_db, test_models + global dev_appserver, testbed - if db is None: + if dev_appserver is None: return - if not os.environ.get('SERVER_SOFTWARE', None): - # this is an extra check because the AppEngine SDK may be in PYTHONPATH - raise unittest.SkipTest('Appengine env not bootstrapped correctly') - # all looks good - we now initialise the imports we require + + from google.appengine.ext import db # noqa from google.appengine.ext import blobstore # noqa from google.appengine.ext.db import polymodel # noqa + from google.appengine.ext import testbed # noqa from pyamf.adapters import _google_appengine_ext_db as adapter_db # noqa from pyamf.adapters import ( # noqa @@ -61,10 +64,16 @@ class BaseTestCase(util.ClassCacheClearingTestCase): """ def setUp(self): - if db is None: + if dev_appserver is None: self.skipTest('google appengine sdk not found') util.ClassCacheClearingTestCase.setUp(self) + self.testbed = testbed.Testbed() + + self.testbed.activate() + # Next, declare which service stubs you want to use. + self.testbed.init_datastore_v3_stub() + self.testbed.init_memcache_stub() def put(self, entity): entity.put() diff --git a/pyamf/tests/gateway/test_django.py b/pyamf/tests/gateway/test_django.py index 32c4bdbb..04f18cbf 100644 --- a/pyamf/tests/gateway/test_django.py +++ b/pyamf/tests/gateway/test_django.py @@ -14,6 +14,12 @@ import os try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +try: + import django as _django from django import http from pyamf.remoting.gateway import django except ImportError: @@ -23,6 +29,22 @@ from pyamf import remoting, util +def make_http_request(method, body=''): + http_request = http.HttpRequest() + http_request.method = method + + version = _django.VERSION[:2] + + if version <= (1, 2): + http_request.raw_post_data = body + else: + http_request._stream = StringIO(body) + # fix a django 1.3 bug where this would not be set + http_request._read_started = False + + return http_request + + class BaseTestCase(unittest.TestCase): """ """ @@ -39,12 +61,14 @@ def setUp(self): import new self.mod_name = '%s.%s' % (__name__, 'settings') - sys.modules[self.mod_name] = new.module(self.mod_name) + self.settings = sys.modules[self.mod_name] = new.module(self.mod_name) self.old_env = os.environ.get('DJANGO_SETTINGS_MODULE', None) os.environ['DJANGO_SETTINGS_MODULE'] = self.mod_name + self.settings.SECRET_KEY = 'unittest' + def tearDown(self): if self.old_env is not None: os.environ['DJANGO_SETTINGS_MODULE'] = self.old_env @@ -78,8 +102,7 @@ def test_settings(self): def test_request_method(self): gw = django.DjangoGateway() - http_request = http.HttpRequest() - http_request.method = 'GET' + http_request = make_http_request('GET') http_response = gw(http_request) self.assertEqual(http_response.status_code, 405) @@ -91,9 +114,7 @@ def test_bad_request(self): request.write('Bad request') request.seek(0, 0) - http_request = http.HttpRequest() - http_request.method = 'POST' - http_request.raw_post_data = request.getvalue() + http_request = make_http_request('POST', request.getvalue()) http_response = gw(http_request) self.assertEqual(http_response.status_code, 400) @@ -109,9 +130,7 @@ def test_unknown_request(self): ) request.seek(0, 0) - http_request = http.HttpRequest() - http_request.method = 'POST' - http_request.raw_post_data = request.getvalue() + http_request = make_http_request('POST', request.getvalue()) http_response = gw(http_request) envelope = remoting.decode(http_response.content) @@ -125,7 +144,6 @@ def test_unknown_request(self): self.assertEqual(body.code, 'Service.ResourceNotFound') def test_expose_request(self): - http_request = http.HttpRequest() self.executed = False def test(request): @@ -142,8 +160,7 @@ def test(request): ) request.seek(0, 0) - http_request.method = 'POST' - http_request.raw_post_data = request.getvalue() + http_request = make_http_request('POST', request.getvalue()) gw(http_request) @@ -158,9 +175,7 @@ def test_really_bad_decode(self): Exception, *args, **kwargs ) - http_request = http.HttpRequest() - http_request.method = 'POST' - http_request.raw_post_data = '' + http_request = make_http_request('POST', '') gw = django.DjangoGateway() @@ -187,9 +202,7 @@ def test_expected_exceptions_decode(self): gw = django.DjangoGateway() - http_request = http.HttpRequest() - http_request.method = 'POST' - http_request.raw_post_data = '' + http_request = make_http_request('POST', '') try: for x in (KeyboardInterrupt, SystemExit): @@ -207,7 +220,6 @@ def test_expected_exceptions_decode(self): def test_timezone(self): import datetime - http_request = http.HttpRequest() self.executed = False td = datetime.timedelta(hours=-5) @@ -228,8 +240,10 @@ def echo(d): msg = remoting.Envelope(amfVersion=pyamf.AMF0) msg['/1'] = remoting.Request(target='test.test', body=[now]) - http_request.method = 'POST' - http_request.raw_post_data = remoting.encode(msg).getvalue() + http_request = make_http_request( + 'POST', + remoting.encode(msg).getvalue() + ) res = remoting.decode(gw(http_request).content) self.assertTrue(self.executed) diff --git a/pyamf/tests/gateway/test_google.py b/pyamf/tests/gateway/test_google.py index 336dc423..02697668 100644 --- a/pyamf/tests/gateway/test_google.py +++ b/pyamf/tests/gateway/test_google.py @@ -10,7 +10,6 @@ """ import unittest -import os from six import StringIO @@ -20,11 +19,6 @@ except ImportError: webapp = None -if os.environ.get('SERVER_SOFTWARE', None) is None: - # we're not being run in appengine environment (at one that we are known to - # work in) - webapp = None - import pyamf from pyamf import remoting @@ -47,7 +41,8 @@ def setUp(self): self.environ = { 'wsgi.input': StringIO(), - 'wsgi.output': StringIO() + 'wsgi.output': StringIO(), + 'webob.is_body_readable': True } self.request = webapp.Request(self.environ) @@ -58,25 +53,26 @@ def setUp(self): def test_get(self): self.gw.get() - self.assertEqual(self.response.__dict__['_Response__status'][0], 405) + self.assertEqual(self.response.status, 405) def test_bad_request(self): self.environ['wsgi.input'].write('Bad request') self.environ['wsgi.input'].seek(0, 0) self.gw.post() - self.assertEqual(self.response.__dict__['_Response__status'][0], 400) + self.assertEqual(self.response.status, 400) def test_unknown_request(self): self.environ['wsgi.input'].write( '\x00\x00\x00\x00\x00\x01\x00\x09test.test\x00\x02/1\x00\x00\x00' '\x14\x0a\x00\x00\x00\x01\x08\x00\x00\x00\x00\x00\x01\x61\x02\x00' - '\x01\x61\x00\x00\x09') + '\x01\x61\x00\x00\x09' + ) self.environ['wsgi.input'].seek(0, 0) self.gw.post() - self.assertEqual(self.response.__dict__['_Response__status'][0], 200) + self.assertEqual(self.response.status, 200) envelope = remoting.decode(self.response.out.getvalue()) message = envelope['/1'] diff --git a/pyamf/tests/test_alias.py b/pyamf/tests/test_alias.py index c0d2011e..b3cc1e28 100644 --- a/pyamf/tests/test_alias.py +++ b/pyamf/tests/test_alias.py @@ -385,13 +385,37 @@ def test_synonym(self): self.assertFalse(self.alias.shortcut_decode) attrs = { - 'foo': 'foo', + 'bar': 'foo', 'spam': 'eggs' } ret = self.alias.getDecodableAttributes(self.obj, attrs) - self.assertEquals(ret, {'bar': 'foo', 'spam': 'eggs'}) + self.assertEquals(ret, {'foo': 'foo', 'spam': 'eggs'}) + + def test_complex_synonym(self): + self.alias.synonym_attrs = {'foo_syn': 'bar_syn'} + self.alias.compile() + + self.alias.static_properties = ['foo_syn', ] + self.alias.exclude_attrs = ['baz', 'gak'] + self.alias.readonly_attrs = ['spam_rd_1', 'spam_rd_2'] + + self.assertFalse(self.alias.shortcut_encode) + self.assertFalse(self.alias.shortcut_decode) + + attrs = { + 'bar_syn': 'foo', + 'spam': 'eggs', + 'spam_rd_1': 'eggs', + 'spam_rd_2': 'eggs', + 'baz': 'remove me', + 'gak': 'remove me' + } + + ret = self.alias.getDecodableAttributes(self.obj, attrs) + + self.assertEquals(ret, {'foo_syn': 'foo', 'spam': 'eggs'}) class ApplyAttributesTestCase(unittest.TestCase): diff --git a/pyamf/tests/test_amf3.py b/pyamf/tests/test_amf3.py index 782bc3db..245cc6ce 100644 --- a/pyamf/tests/test_amf3.py +++ b/pyamf/tests/test_amf3.py @@ -1540,7 +1540,7 @@ def build_complex(self, max=5): return test_objects - def complex_test(self): + def complex_trial(self): to_cd = self.context.getClass(self.TestObject) tso_cd = self.context.getClass(self.TestSubObject) @@ -1558,7 +1558,7 @@ def test_complex_dict(self): complex = {'element': 'ignore', 'objects': self.build_complex()} self.encoder.writeElement(complex) - self.complex_test() + self.complex_trial() def test_complex_encode_decode_dict(self): complex = {'element': 'ignore', 'objects': self.build_complex()} diff --git a/setup.cfg b/setup.cfg index 5514a2b4..4bff0526 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,4 +51,4 @@ sourcecode: yes #dotpath: /usr/local/bin/dot [flake8] -exclude=doc,.ropeproject,distribute_setup.py +exclude=doc,.ropeproject diff --git a/setup.py b/setup.py index 4c78767a..b46cf8c3 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ name = "PyAMF" description = "AMF support for Python" -long_description = setupinfo.read('README.txt') +long_description = setupinfo.read('README.rst') url = "http://pyamf.org" author = "The PyAMF Project" author_email = "users@pyamf.org"