update to a newer version of magic.py, fix bugs in minimize and codeg…

…en related to decorator handling, make sure magic.py globals don't get munged in un.rpyc and general bugfixes

decompiler/codegen.py

@@ -306,6 +306,7 @@ def write_comma():
 
     def decorators(self, node):
         for decorator in node.decorator_list:
+            self.force_newline = True
             self.newline(decorator)
             self.write('@')
             self.visit(decorator)

decompiler/magic.py

@@ -223,80 +223,125 @@ def __subclasscheck__(self, subclass):
 
 
 # Default FakeClass instance methods
-
-def _strict_new(cls, *args, **kwargs):
-    self = cls.__bases__[0].__new__(cls)
-    if args:
-        raise FakeUnpicklingError("{0} was instantiated with unexpected arguments {1}, {2}".format(cls, args, kwargs))
-    return self
-
-def _warning_new(cls, *args, **kwargs):
-    self = cls.__bases__[0].__new__(cls)
-    if args:
-        print("{0} was instantiated with unexpected arguments {1}, {2}".format(cls, args, kwargs))
-        self._new_args = args
-    return self
-
-def _ignore_new(cls, *args, **kwargs):
-    self = cls.__bases__[0].__new__(cls)
-    if args:
-        self._new_args = args
-    return self
-
-def _strict_setstate(self, state):
-    slotstate = None
-
-    if (isinstance(state, tuple) and len(state) == 2 and
-        (state[0] is None or isinstance(state[0], dict)) and
-        (state[1] is None or isinstance(state[1], dict))):
-        state, slotstate = state
-
-    if state:
-        # Don't have to check for slotstate here since it's either None or a dict
-        if not isinstance(state, dict):
-            raise FakeUnpicklingError("{0}.__setstate__() got unexpected arguments {1}".format(self.__class__, state))
+class FakeClassTemplateMeta(type):
+    def __new__(cls, name, bases, attributes, module=None):
+        # don't need this
+        attributes.pop("__qualname__", None)
+
+        # find bases
+        actbases = []
+        for base in bases:
+            if isinstance(base, FakeClassTemplateMeta):
+                actbases.extend(base.bases)
+            else:
+                actbases.append(base)
+
+        # and module
+        # note that if no module is explicitly passed, the current module will be chosen
+        if module is None:
+            module = attributes.pop("__module__", None)
+            if module is None:
+                raise TypeError("No module has been specified for FakeClassTemplate {0}".format(name))
         else:
-            self.__dict__.update(state)
-
-    if slotstate:
-        self.__dict__.update(slotstate)
+            attributes.pop("__module__", None)
 
-def _warning_setstate(self, state):
-    slotstate = None
-
-    if (isinstance(state, tuple) and len(state) == 2 and
-        (state[0] is None or isinstance(state[0], dict)) and
-        (state[1] is None or isinstance(state[1], dict))):
-        state, slotstate = state
-
-    if state:
-        # Don't have to check for slotstate here since it's either None or a dict
-        if not isinstance(state, dict):
-            print("{0}.__setstate__() got unexpected arguments {1}".format(self.__class__, state))
-            self._setstate_args = state
-        else:
-            self.__dict__.update(state)
+        # assemble instance
+        self = type.__new__(cls, name, (object,), {"bases": tuple(actbases), "attributes": attributes, "__module__": module})
 
-    if slotstate:
-        self.__dict__.update(slotstate)
+        return self
 
-def _ignore_setstate(self, state):
-    slotstate = None
+    def __init__(self, name, bases, attributes, module=None):
+        pass
 
-    if (isinstance(state, tuple) and len(state) == 2 and
-        (state[0] is None or isinstance(state[0], dict)) and
-        (state[1] is None or isinstance(state[1], dict))):
-        state, slotstate = state
+    def __call__(self):
+        # instantiating these things makes rather little sense
+        return NotImplementedError("Cannot instantiate a fake class template")
 
-    if state:
-        # Don't have to check for slotstate here since it's either None or a dict
-        if not isinstance(state, dict):
-            self._setstate_args = state
-        else:
-            self.__dict__.update(state)
+    def __repr__(self):
+        return "<FakeClassTemplate '{0}.{1}'>".format(self.__module__, self.__name__)
+
+# PY2 doesn't like the PY3 way of metaclasses and PY3 doesn't support the PY2 way
+# so we call the metaclass directly
+FakeClassTemplate = FakeClassTemplateMeta("fake_class_template", (), {"__module__": __name__})
+
+class _strict(FakeClassTemplate):
+    def __new__(cls, *args, **kwargs):
+        self = super(cls, cls).__new__(cls)
+        if args or kwargs:
+            raise FakeUnpicklingError("{0} was instantiated with unexpected arguments {1}, {2}".format(cls, args, kwargs))
+        return self
+
+    def __setstate__(self, state):
+        slotstate = None
+
+        if (isinstance(state, tuple) and len(state) == 2 and
+            (state[0] is None or isinstance(state[0], dict)) and
+            (state[1] is None or isinstance(state[1], dict))):
+            state, slotstate = state
+
+        if state:
+            # Don't have to check for slotstate here since it's either None or a dict
+            if not isinstance(state, dict):
+                raise FakeUnpicklingError("{0}.__setstate__() got unexpected arguments {1}".format(self.__class__, state))
+            else:
+                self.__dict__.update(state)
+
+        if slotstate:
+            self.__dict__.update(slotstate)
+
+class _warning(FakeClassTemplate):
+    def __new__(cls, *args, **kwargs):
+        self = super(cls, cls).__new__(cls)
+        if args or kwargs:
+            print("{0} was instantiated with unexpected arguments {1}, {2}".format(cls, args, kwargs))
+            self._new_args = args
+        return self
+
+    def __setstate__(self, state):
+        slotstate = None
+
+        if (isinstance(state, tuple) and len(state) == 2 and
+            (state[0] is None or isinstance(state[0], dict)) and
+            (state[1] is None or isinstance(state[1], dict))):
+            state, slotstate = state
+
+        if state:
+            # Don't have to check for slotstate here since it's either None or a dict
+            if not isinstance(state, dict):
+                print("{0}.__setstate__() got unexpected arguments {1}".format(self.__class__, state))
+                self._setstate_args = state
+            else:
+                self.__dict__.update(state)
+
+        if slotstate:
+            self.__dict__.update(slotstate)
+
+class _ignore(FakeClassTemplate):
+    def __new__(cls, *args, **kwargs):
+        self = super(cls, cls).__new__(cls)
+        if args:
+            self._new_args = args
+        if kwargs:
+            self._new_kwargs = kwargs
+        return self
+
+    def __setstate__(self, state):
+        slotstate = None
+
+        if (isinstance(state, tuple) and len(state) == 2 and
+            (state[0] is None or isinstance(state[0], dict)) and
+            (state[1] is None or isinstance(state[1], dict))):
+            state, slotstate = state
+
+        if state:
+            # Don't have to check for slotstate here since it's either None or a dict
+            if not isinstance(state, dict):
+                self._setstate_args = state
+            else:
+                self.__dict__.update(state)
 
-    if slotstate:
-        self.__dict__.update(slotstate)
+        if slotstate:
+            self.__dict__.update(slotstate)
 
 class FakeClassFactory(object):
     """
@@ -329,31 +374,34 @@ def __init__(self, special_cases, errors='strict', fake_metaclass=FakeClassType,
         but if old-style classes are desired in Python 2, it can be set to an empty tuple.
 
         both the default methods and default bases can be overridden using *special_cases*,
-        which should follow this syntax: ``special_cases = {"module.name": (bases, methods)``
-        in which bases is a tuple of classes to inherit from and methods is a dictionary of
-        attribute name to value. In case value is a function, it will be used as a bound method.
+        which should be a list of :class:`FakeClassTemplate` instances. The methods and bases defined
+        on the :class:`FakeClassTemplate` instances will then be used to instantiate fake classes with
+        matching name and module.
 
         As an example, we can define the fake class generated for definition bar in module foo,
-        which has a :meth:`__str__` method which returns ``"baz"``:
+        which has a :meth:`__str__` method which returns ``"baz"``::
+
+           class bar(object, fake_class_template):
+               def __str__(self): return "baz"
 
-        ``special_cases = {"foo.bar": ((object, ), {"__str__": (lambda self: "baz")})}``
+           special_cases = [bar]
 
-        Finally it can be noted that the equivalent of another class can be generated using:
+        Alternatively they can also be instantiated programmatically like in:
 
-        ``special_cases = {cls.__module__ + "." + cls.__name__: (cls.__bases__, cls.__dict__)}``
+        ``special_cases = [FakeClassTemplate(c.__name__, c.__bases__, c.__dict__, c.__module__)]``
         """
-        self.special_cases = special_cases
+        self.special_cases = dict(((i.__module__, i.__name__), i) for i in special_cases)
         self.metaclass = fake_metaclass
         self.default_bases = default_bases
 
         self.class_cache = {}
 
         if errors == 'strict':
-            self.default_attributes = {"__new__": _strict_new, "__setstate__": _strict_setstate}
+            self.default_attributes = _strict.attributes
         elif errors == 'warning':
-            self.default_attributes = {"__new__": _warning_new, "__setstate__": _warning_setstate}
+            self.default_attributes = _strict.attributes
         elif errors == 'ignore':
-            self.default_attributes = {"__new__": _ignore_new, "__setstate__": _ignore_setstate}
+            self.default_attributes = _strict.attributes
         else:
             raise ValueError("Unknown error handling directive '{0}' given".format(errors))
 
@@ -372,11 +420,11 @@ def __call__(self, name, module):
         if klass is not None:
             return klass
 
-        special = self.special_cases.get(module + "." + name, None)
+        special = self.special_cases.get((module, name), None)
 
         attributes = self.default_attributes.copy()
         if special:
-            bases, new_attributes = special
+            bases, new_attributes = special.bases, special.attributes
             attributes.update(new_attributes)
         else:
             bases = self.default_bases

un.rpyc/compile.py

@@ -52,12 +52,12 @@
 except ImportError:
     exit("Could not import pickleast. Are you sure it's in pythons module search path?")
 
-def Module(name, filename):
+def Module(name, filename, munge_globals=True):
     with open(filename, "rb" if p.PY2 else "r") as f:
         code = f.read()
     if args.minimize:
         # in modules only locals are worth optimizing
-        code = minimize.minimize(code, True, args.obfuscate, args.obfuscate, args.obfuscate)
+        code = minimize.minimize(code, True, args.obfuscate and munge_globals, args.obfuscate, args.obfuscate)
     return p.Module(name, code, False)
 
 def Exec(code):
@@ -94,7 +94,7 @@ def Exec(code):
             sys.files.append((abspath, fn, dir, fobj))
 """),
                     Module("util", path.join(base_folder, "decompiler/util.py")),
-                    Module("magic", path.join(base_folder, "decompiler/magic.py")),
+                    Module("magic", path.join(base_folder, "decompiler/magic.py"), False),
                     Module("codegen", path.join(base_folder, "decompiler/codegen.py")),
                     p.Assign("renpy_modules", p.Imports("sys", "modules").copy()),
                     Exec("""

un.rpyc/minimize.py

@@ -297,6 +297,13 @@ def analyze(self, node, protect_globals, protect_builtins, protect_imports, mung
         return node
 
     def scoped_visit(self, node, type, protect=False):
+        # decorators, although they are a part of this node
+        # reside in the enclosing scope. so we parse them before and then temporarily
+        # empty the decorator_list
+        decorators = []
+        for decorator in node.decorator_list:
+            decorators.append(self.visit(decorator))
+        node.decorator_list = []
         if self.stage == self.ANALYZE:
             self.scope = self.scope.child(type, protect)
             node._scope = self.scope
@@ -307,6 +314,7 @@ def scoped_visit(self, node, type, protect=False):
             node = self.generic_visit(node)
             self.scope = self.scope.parent
             del node._scope
+        node.decorator_list = decorators
         return node
 
     def new_name(self, name):

un.rpyc/unrpyc-compile.py

@@ -28,18 +28,21 @@
 
 # special new and setstate methods for special classes
 
-def PyExprNew(cls, s, filename, linenumber):
-    self = unicode.__new__(cls, s)
-    self.filename = filename
-    self.linenumber = linenumber
-    return self
-
-def PyCodeSetstate(self, state):
-    (_, self.source, self.location, self.mode) = state
-    self.bytecode = None
-
-factory = magic.FakeClassFactory({"renpy.ast.PyExpr": ((unicode,), {"__new__": PyExprNew}),
-                                  "renpy.ast.PyCode": ((object,), {"__setstate__": PyCodeSetstate})})
+class PyExpr(unicode, magic.FakeClassTemplate):
+    __module__ = "renpy.ast"
+    def __new__(cls, s, filename, linenumber):
+        self = unicode.__new__(cls, s)
+        self.filename = filename
+        self.linenumber = linenumber
+        return self
+
+class PyCode(object, magic.FakeClassTemplate):
+    __module__ = "renpy.ast"
+    def __setstate__(self, state):
+        (_, self.source, self.location, self.mode) = state
+        self.bytecode = None
+
+factory = magic.FakeClassFactory((PyExpr, PyCode))
 
 def read_ast_from_file(in_file):
     # .rpyc files are just zlib compressed pickles of a tuple of some data and the actual AST of the file

unrpyc.py

@@ -30,22 +30,23 @@
 import decompiler
 from decompiler import magic, astdump
 
-# special new and setstate methods for special classes
-
-def PyExprNew(cls, s, filename, linenumber):
-    self = unicode.__new__(cls, s)
-    self.filename = filename
-    self.linenumber = linenumber
-    return self
-
-def PyCodeSetstate(self, state):
-    (_, self.source, self.location, self.mode) = state
-    self.bytecode = None
-
-class_factory = magic.FakeClassFactory({
-    "renpy.ast.PyExpr": ((unicode,), {"__new__": PyExprNew}),
-    "renpy.ast.PyCode": ((object,), {"__setstate__": PyCodeSetstate})
-})
+# special definitions for special classes
+
+class PyExpr(unicode, magic.FakeClassTemplate):
+    __module__ = "renpy.ast"
+    def __new__(cls, s, filename, linenumber):
+        self = unicode.__new__(cls, s)
+        self.filename = filename
+        self.linenumber = linenumber
+        return self
+
+class PyCode(object, magic.FakeClassTemplate):
+    __module__ = "renpy.ast"
+    def __setstate__(self, state):
+        (_, self.source, self.location, self.mode) = state
+        self.bytecode = None
+
+class_factory = magic.FakeClassFactory((PyExpr, PyCode))
 
 # API