rgcombine.py

import argparse
import ast
import base64
import cStringIO
import os
import pickle
import sys
import zlib

from contextlib import contextmanager

def get_arg_parser():
  parser = argparse.ArgumentParser(
    description = """Python import combine script.

This script takes as input a python file that imports other python files in the
same directory, and combines them into one python file.""",
    formatter_class = argparse.RawTextHelpFormatter,
  )
  parser.add_argument(
    'input_path',
    help = 'path to python file to be combined',
  )
  parser.add_argument(
    'output_path',
    help = 'path of file to write output to',
  )
  parser.add_argument("-s", "--short", action="store_true",
                        default=False,
                        help="Shorten output but give up exception info.",
  )
  parser.add_argument("-f", "--force", action="store_true",
                        default=False,
                        help="Ignore version difference and run anyway.",
  )
  return parser

def warning(message):
  sys.stderr.write('Warning: ' + message + '\n')

@contextmanager
def chdir(directory):
  orig_directory = os.getcwd()
  os.chdir(directory)
  yield
  os.chdir(orig_directory)

def module_to_file(mod_name):
  file_name = mod_name + '.py'
  if os.path.isfile(file_name):
    return file_name
  else:
    return None

def file_to_module(file_name):
  if len(file_name) < 4 or file_name[-3:] != '.py':
    raise Exception('File name %s does not give a valid module name!')
  return file_name[:-3]

module_dict_name = '__moduledict__'
code_dict_name = '__codedict__'
root_code_name = '__rootcode__'
temp_dict_name = '__tempdict__'
module_class_name = '__FakeModule__'

class ModuleFinder(ast.NodeVisitor):
  def __init__(self):
    ast.NodeVisitor.__init__(self)
    self._modules = []

  def run(self, tree):
    self.visit(tree)
    return self._modules

  def visit_Import(self, node):
    for alias in node.names:
      self._modules.append(alias.name)
  
  def visit_ImportFrom(self, node):
    self._modules.append(node.module)

def get_module_trees(root_tree):
  module_trees = {}
  modules = ModuleFinder().run(root_tree)
  while modules:
    module = modules.pop()
    if module not in module_trees:
      file_name = module_to_file(module)
      if file_name is not None:
        with open(file_name, 'r') as fd:
          source = fd.read()
        tree = ast.parse(source)
        module_trees[module] = tree
        modules.extend(ModuleFinder().run(tree))
  return module_trees

class ImportTransformer(ast.NodeTransformer):
  def __init__(self, modules):
    ast.NodeTransformer.__init__(self)
    self._modules = modules

  def visit_Import(self, node):
    nodes = []
    for alias in node.names:
      module = alias.name
      asname = alias.asname if alias.asname else module
      if module in self._modules:
        nodes.extend(self._load_module_nodes(module))
        code = "%s=%s['%s']" % (asname, module_dict_name, module)
        tree = ast.parse(code)
        nodes.extend(tree.body)
      else:
        nodes.append(ast.Import(names=[alias]))
    for n in nodes:
      self._set_locations(n, node.lineno, node.col_offset)
    return nodes

  def visit_ImportFrom(self, node):
    module = node.module
    if node.level != 0:
      warning("Absolute import detected! They are not yet supported.")
    elif module in self._modules:
      nodes = []
      nodes.extend(self._load_module_nodes(module))
      code = ""
      for alias in node.names:
        if alias.name == '*':
          code += "locals().update(%s['%s'].__dict__)\n" % (
            module_dict_name,
            module
          )
        else:
          asname = alias.asname if alias.asname else alias.name
          code += "locals()['%s'] = %s['%s'].__dict__['%s']\n" % (
            asname,
            module_dict_name,
            module,
            alias.name
          )
      tree = ast.parse(code)
      nodes.extend(tree.body)
      for n in nodes:
        self._set_locations(n, node.lineno, node.col_offset)
      return nodes
    else:
      return node

  def _load_module_nodes(self, module):
    code = """
if '%s' not in %s:
  %s['%s']=type('%s',(object,),{})()
  %s['%s'].__dict__={'%s':%s,'%s':%s}
  exec %s['%s'] in %s['%s'].__dict__
""" % (module,
       module_dict_name,
       module_dict_name,
       module,
       module_class_name,
       module_dict_name,
       module,
       module_dict_name,
       module_dict_name,
       code_dict_name,
       code_dict_name,
       code_dict_name,
       module,
       module_dict_name,
       module)
    tree = ast.parse(code)
    return tree.body

  def _set_locations(self, node, lineno, col_offset):
    for n in ast.walk(node):
      n.lineno = lineno
      n.col_offset = col_offset

def main():
  args = get_arg_parser().parse_args()
  if sys.version_info < (2, 7) or sys.version_info >= (2, 8):
    if not args.force:
      raise Exception("Python version not supported! Use Python 2.7.")
  input_path = os.path.abspath(args.input_path)
  output_path = os.path.abspath(args.output_path)
  with chdir(os.path.dirname(input_path)):
    file_name = os.path.basename(input_path)
    with open(file_name, 'r') as fd:
      source = fd.read()
    root_tree = ast.parse(source, filename=file_name)
    module_trees = get_module_trees(root_tree)
    modules = module_trees.keys()
    for module in module_trees:
      ImportTransformer(modules).visit(module_trees[module])
      if args.short:
        strio = cStringIO.StringIO()
        Unparser(module_trees[module], strio)
        module_trees[module] = strio.getvalue()
        strio.close()
      else:
        module_trees[module] = pickle.dumps(module_trees[module])
      module_trees[module] = zlib.compress(module_trees[module], 9)
      module_trees[module] = base64.b64encode(module_trees[module])
    output = "import pickle,base64,zlib\n"
    output += "%s={}\n" % module_dict_name
    output += "%s={}\n" % code_dict_name
    for module, encoded_tree in module_trees.iteritems():
      if args.short:
        line = "'%s'" % encoded_tree
        line = "base64.decodestring(%s)" % line
        line = "zlib.decompress(%s)" % line
        line = "compile(%s,'%s','exec')" % (line, module_to_file(module))
        line = "%s['%s']=%s" % (code_dict_name, module, line)
      else:
        line = "'%s'" % encoded_tree
        line = "base64.decodestring(%s)" % line
        line = "zlib.decompress(%s)" % line
        line = "pickle.loads(%s)" % line
        line = "compile(%s,'%s','exec')" % (line, module_to_file(module))
        line = "%s['%s']=%s" % (code_dict_name, module, line)
      output += "%s\n" % line
    ImportTransformer(modules).visit(root_tree)
    root_tree = pickle.dumps(root_tree)
    root_tree = zlib.compress(root_tree)
    root_tree = base64.b64encode(root_tree)
    line = "'%s'" % root_tree
    line = "base64.decodestring(%s)" % line
    line = "zlib.decompress(%s)" % line
    line = "pickle.loads(%s)" % line
    line = "compile(%s,'%s','exec')" % (line, file_name)
    line = "%s=%s" % (root_code_name, line)
    output += "%s\n" % line
    output += "del pickle,base64,zlib\n"
    output += "exec %s" % root_code_name
    with open(output_path, 'w') as fd:
      fd.write(output)

"""
Copy and pasted unparse.py
"""

# Large float and imaginary literals get turned into infinities in the AST.
# We unparse those infinities to INFSTR.
INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)

def interleave(inter, f, seq):
  """Call f on each item in seq, calling inter() in between.
  """
  seq = iter(seq)
  try:
    f(next(seq))
  except StopIteration:
    pass
  else:
    for x in seq:
      inter()
      f(x)

class Unparser:
  """Methods in this class recursively traverse an AST and
  output source code for the abstract syntax; original formatting
  is disregarded. """

  def __init__(self, tree, file = sys.stdout):
    """Unparser(tree, file=sys.stdout) -> None.
     Print the source for tree to file."""
    self.f = file
    self.future_imports = []
    self._indent = 0
    self.dispatch(tree)
    self.f.write("")
    self.f.flush()

  def fill(self, text = ""):
    "Indent a piece of text, according to the current indentation level"
    self.f.write("\n"+"  "*self._indent + text)

  def write(self, text):
    "Append a piece of text to the current line."
    self.f.write(text)

  def enter(self):
    "Print ':', and increase the indentation."
    self.write(":")
    self._indent += 1

  def leave(self):
    "Decrease the indentation level."
    self._indent -= 1

  def dispatch(self, tree):
    "Dispatcher function, dispatching tree type T to method _T."
    if isinstance(tree, list):
      for t in tree:
        self.dispatch(t)
      return
    meth = getattr(self, "_"+tree.__class__.__name__)
    meth(tree)


  ############### Unparsing methods ######################
  # There should be one method per concrete grammar type #
  # Constructors should be grouped by sum type. Ideally, #
  # this would follow the order in the grammar, but   #
  # currently doesn't.                  #
  ########################################################

  def _Module(self, tree):
    for stmt in tree.body:
      self.dispatch(stmt)

  # stmt
  def _Expr(self, tree):
    self.fill()
    self.dispatch(tree.value)

  def _Import(self, t):
    self.fill("import ")
    interleave(lambda: self.write(", "), self.dispatch, t.names)

  def _ImportFrom(self, t):
    # A from __future__ import may affect unparsing, so record it.
    if t.module and t.module == '__future__':
      self.future_imports.extend(n.name for n in t.names)

    self.fill("from ")
    self.write("." * t.level)
    if t.module:
      self.write(t.module)
    self.write(" import ")
    interleave(lambda: self.write(", "), self.dispatch, t.names)

  def _Assign(self, t):
    self.fill()
    for target in t.targets:
      self.dispatch(target)
      self.write(" = ")
    self.dispatch(t.value)

  def _AugAssign(self, t):
    self.fill()
    self.dispatch(t.target)
    self.write(" "+self.binop[t.op.__class__.__name__]+"= ")
    self.dispatch(t.value)

  def _Return(self, t):
    self.fill("return")
    if t.value:
      self.write(" ")
      self.dispatch(t.value)

  def _Pass(self, t):
    self.fill("pass")

  def _Break(self, t):
    self.fill("break")

  def _Continue(self, t):
    self.fill("continue")

  def _Delete(self, t):
    self.fill("del ")
    interleave(lambda: self.write(", "), self.dispatch, t.targets)

  def _Assert(self, t):
    self.fill("assert ")
    self.dispatch(t.test)
    if t.msg:
      self.write(", ")
      self.dispatch(t.msg)

  def _Exec(self, t):
    self.fill("exec ")
    self.dispatch(t.body)
    if t.globals:
      self.write(" in ")
      self.dispatch(t.globals)
    if t.locals:
      self.write(", ")
      self.dispatch(t.locals)

  def _Print(self, t):
    self.fill("print ")
    do_comma = False
    if t.dest:
      self.write(">>")
      self.dispatch(t.dest)
      do_comma = True
    for e in t.values:
      if do_comma:self.write(", ")
      else:do_comma=True
      self.dispatch(e)
    if not t.nl:
      self.write(",")

  def _Global(self, t):
    self.fill("global ")
    interleave(lambda: self.write(", "), self.write, t.names)

  def _Yield(self, t):
    self.write("(")
    self.write("yield")
    if t.value:
      self.write(" ")
      self.dispatch(t.value)
    self.write(")")

  def _Raise(self, t):
    self.fill('raise ')
    if t.type:
      self.dispatch(t.type)
    if t.inst:
      self.write(", ")
      self.dispatch(t.inst)
    if t.tback:
      self.write(", ")
      self.dispatch(t.tback)

  def _TryExcept(self, t):
    self.fill("try")
    self.enter()
    self.dispatch(t.body)
    self.leave()

    for ex in t.handlers:
      self.dispatch(ex)
    if t.orelse:
      self.fill("else")
      self.enter()
      self.dispatch(t.orelse)
      self.leave()

  def _TryFinally(self, t):
    if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept):
      # try-except-finally
      self.dispatch(t.body)
    else:
      self.fill("try")
      self.enter()
      self.dispatch(t.body)
      self.leave()

    self.fill("finally")
    self.enter()
    self.dispatch(t.finalbody)
    self.leave()

  def _ExceptHandler(self, t):
    self.fill("except")
    if t.type:
      self.write(" ")
      self.dispatch(t.type)
    if t.name:
      self.write(" as ")
      self.dispatch(t.name)
    self.enter()
    self.dispatch(t.body)
    self.leave()

  def _ClassDef(self, t):
    self.write("\n")
    for deco in t.decorator_list:
      self.fill("@")
      self.dispatch(deco)
    self.fill("class "+t.name)
    if t.bases:
      self.write("(")
      for a in t.bases:
        self.dispatch(a)
        self.write(", ")
      self.write(")")
    self.enter()
    self.dispatch(t.body)
    self.leave()

  def _FunctionDef(self, t):
    self.write("\n")
    for deco in t.decorator_list:
      self.fill("@")
      self.dispatch(deco)
    self.fill("def "+t.name + "(")
    self.dispatch(t.args)
    self.write(")")
    self.enter()
    self.dispatch(t.body)
    self.leave()

  def _For(self, t):
    self.fill("for ")
    self.dispatch(t.target)
    self.write(" in ")
    self.dispatch(t.iter)
    self.enter()
    self.dispatch(t.body)
    self.leave()
    if t.orelse:
      self.fill("else")
      self.enter()
      self.dispatch(t.orelse)
      self.leave()

  def _If(self, t):
    self.fill("if ")
    self.dispatch(t.test)
    self.enter()
    self.dispatch(t.body)
    self.leave()
    # collapse nested ifs into equivalent elifs.
    while (t.orelse and len(t.orelse) == 1 and
        isinstance(t.orelse[0], ast.If)):
      t = t.orelse[0]
      self.fill("elif ")
      self.dispatch(t.test)
      self.enter()
      self.dispatch(t.body)
      self.leave()
    # final else
    if t.orelse:
      self.fill("else")
      self.enter()
      self.dispatch(t.orelse)
      self.leave()

  def _While(self, t):
    self.fill("while ")
    self.dispatch(t.test)
    self.enter()
    self.dispatch(t.body)
    self.leave()
    if t.orelse:
      self.fill("else")
      self.enter()
      self.dispatch(t.orelse)
      self.leave()

  def _With(self, t):
    self.fill("with ")
    self.dispatch(t.context_expr)
    if t.optional_vars:
      self.write(" as ")
      self.dispatch(t.optional_vars)
    self.enter()
    self.dispatch(t.body)
    self.leave()

  # expr
  def _Str(self, tree):
    # if from __future__ import unicode_literals is in effect,
    # then we want to output string literals using a 'b' prefix
    # and unicode literals with no prefix.
    if "unicode_literals" not in self.future_imports:
      self.write(repr(tree.s))
    elif isinstance(tree.s, str):
      self.write("b" + repr(tree.s))
    elif isinstance(tree.s, unicode):
      self.write(repr(tree.s).lstrip("u"))
    else:
      assert False, "shouldn't get here"

  def _Name(self, t):
    self.write(t.id)

  def _Repr(self, t):
    self.write("`")
    self.dispatch(t.value)
    self.write("`")

  def _Num(self, t):
    repr_n = repr(t.n)
    # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2.
    if repr_n.startswith("-"):
      self.write("(")
    # Substitute overflowing decimal literal for AST infinities.
    self.write(repr_n.replace("inf", INFSTR))
    if repr_n.startswith("-"):
      self.write(")")

  def _List(self, t):
    self.write("[")
    interleave(lambda: self.write(", "), self.dispatch, t.elts)
    self.write("]")

  def _ListComp(self, t):
    self.write("[")
    self.dispatch(t.elt)
    for gen in t.generators:
      self.dispatch(gen)
    self.write("]")

  def _GeneratorExp(self, t):
    self.write("(")
    self.dispatch(t.elt)
    for gen in t.generators:
      self.dispatch(gen)
    self.write(")")

  def _SetComp(self, t):
    self.write("{")
    self.dispatch(t.elt)
    for gen in t.generators:
      self.dispatch(gen)
    self.write("}")

  def _DictComp(self, t):
    self.write("{")
    self.dispatch(t.key)
    self.write(": ")
    self.dispatch(t.value)
    for gen in t.generators:
      self.dispatch(gen)
    self.write("}")

  def _comprehension(self, t):
    self.write(" for ")
    self.dispatch(t.target)
    self.write(" in ")
    self.dispatch(t.iter)
    for if_clause in t.ifs:
      self.write(" if ")
      self.dispatch(if_clause)

  def _IfExp(self, t):
    self.write("(")
    self.dispatch(t.body)
    self.write(" if ")
    self.dispatch(t.test)
    self.write(" else ")
    self.dispatch(t.orelse)
    self.write(")")

  def _Set(self, t):
    assert(t.elts) # should be at least one element
    self.write("{")
    interleave(lambda: self.write(", "), self.dispatch, t.elts)
    self.write("}")

  def _Dict(self, t):
    self.write("{")
    def write_pair(pair):
      (k, v) = pair
      self.dispatch(k)
      self.write(": ")
      self.dispatch(v)
    interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values))
    self.write("}")

  def _Tuple(self, t):
    self.write("(")
    if len(t.elts) == 1:
      (elt,) = t.elts
      self.dispatch(elt)
      self.write(",")
    else:
      interleave(lambda: self.write(", "), self.dispatch, t.elts)
    self.write(")")

  unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"}
  def _UnaryOp(self, t):
    self.write("(")
    self.write(self.unop[t.op.__class__.__name__])
    self.write(" ")
    # If we're applying unary minus to a number, parenthesize the number.
    # This is necessary: -2147483648 is different from -(2147483648) on
    # a 32-bit machine (the first is an int, the second a long), and
    # -7j is different from -(7j). (The first has real part 0.0, the second
    # has real part -0.0.)
    if isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num):
      self.write("(")
      self.dispatch(t.operand)
      self.write(")")
    else:
      self.dispatch(t.operand)
    self.write(")")

  binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%",
          "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&",
          "FloorDiv":"//", "Pow": "**"}
  def _BinOp(self, t):
    self.write("(")
    self.dispatch(t.left)
    self.write(" " + self.binop[t.op.__class__.__name__] + " ")
    self.dispatch(t.right)
    self.write(")")

  cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=",
            "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"}
  def _Compare(self, t):
    self.write("(")
    self.dispatch(t.left)
    for o, e in zip(t.ops, t.comparators):
      self.write(" " + self.cmpops[o.__class__.__name__] + " ")
      self.dispatch(e)
    self.write(")")

  boolops = {ast.And: 'and', ast.Or: 'or'}
  def _BoolOp(self, t):
    self.write("(")
    s = " %s " % self.boolops[t.op.__class__]
    interleave(lambda: self.write(s), self.dispatch, t.values)
    self.write(")")

  def _Attribute(self,t):
    self.dispatch(t.value)
    # Special case: 3.__abs__() is a syntax error, so if t.value
    # is an integer literal then we need to either parenthesize
    # it or add an extra space to get 3 .__abs__().
    if isinstance(t.value, ast.Num) and isinstance(t.value.n, int):
      self.write(" ")
    self.write(".")
    self.write(t.attr)

  def _Call(self, t):
    self.dispatch(t.func)
    self.write("(")
    comma = False
    for e in t.args:
      if comma: self.write(", ")
      else: comma = True
      self.dispatch(e)
    for e in t.keywords:
      if comma: self.write(", ")
      else: comma = True
      self.dispatch(e)
    if t.starargs:
      if comma: self.write(", ")
      else: comma = True
      self.write("*")
      self.dispatch(t.starargs)
    if t.kwargs:
      if comma: self.write(", ")
      else: comma = True
      self.write("**")
      self.dispatch(t.kwargs)
    self.write(")")

  def _Subscript(self, t):
    self.dispatch(t.value)
    self.write("[")
    self.dispatch(t.slice)
    self.write("]")

  # slice
  def _Ellipsis(self, t):
    self.write("...")

  def _Index(self, t):
    self.dispatch(t.value)

  def _Slice(self, t):
    if t.lower:
      self.dispatch(t.lower)
    self.write(":")
    if t.upper:
      self.dispatch(t.upper)
    if t.step:
      self.write(":")
      self.dispatch(t.step)

  def _ExtSlice(self, t):
    interleave(lambda: self.write(', '), self.dispatch, t.dims)

  # others
  def _arguments(self, t):
    first = True
    # normal arguments
    defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults
    for a,d in zip(t.args, defaults):
      if first:first = False
      else: self.write(", ")
      self.dispatch(a),
      if d:
        self.write("=")
        self.dispatch(d)

    # varargs
    if t.vararg:
      if first:first = False
      else: self.write(", ")
      self.write("*")
      self.write(t.vararg)

    # kwargs
    if t.kwarg:
      if first:first = False
      else: self.write(", ")
      self.write("**"+t.kwarg)

  def _keyword(self, t):
    self.write(t.arg)
    self.write("=")
    self.dispatch(t.value)

  def _Lambda(self, t):
    self.write("(")
    self.write("lambda ")
    self.dispatch(t.args)
    self.write(": ")
    self.dispatch(t.body)
    self.write(")")

  def _alias(self, t):
    self.write(t.name)
    if t.asname:
      self.write(" as "+t.asname)

if __name__ == '__main__':
  main()