From c6064633b99def9deecb063269599c3f122e5531 Mon Sep 17 00:00:00 2001 From: Jackmcbarn Date: Mon, 16 Nov 2020 02:22:18 -0500 Subject: [PATCH 01/18] Support Ren'Py commit 65a36a4 --- decompiler/translate.py | 58 ++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/decompiler/translate.py b/decompiler/translate.py index 3365c772..872993b1 100644 --- a/decompiler/translate.py +++ b/decompiler/translate.py @@ -32,6 +32,29 @@ def __init__(self, language, saving_translations=False): self.strings = {} self.dialogue = {} self.identifiers = set() + self.alternate = None + + # Adapted from Ren'Py's Restructurer.unique_identifier + def unique_identifier(self, label, digest): + if label is None: + base = digest + else: + base = label.replace(".", "_") + "_" + digest + + i = 0 + suffix = "" + + while True: + + identifier = base + suffix + + if identifier not in self.identifiers: + break + + i += 1 + suffix = "_{0}".format(i) + + return identifier # Adapted from Ren'Py's Restructurer.create_translate def create_translate(self, block): @@ -49,27 +72,20 @@ def create_translate(self, block): raise Exception("Don't know how to get canonical code for a %s" % str(type(i))) md5.update(code.encode("utf-8") + b"\r\n") - if self.label: - base = self.label + "_" + md5.hexdigest()[:8] - else: - base = md5.hexdigest()[:8] - - i = 0 - suffix = "" - - while True: - - identifier = base + suffix - - if identifier not in self.identifiers: - break - - i += 1 - suffix = "_{0}".format(i) + digest = md5.hexdigest()[:8] + identifier = self.unique_identifier(self.label, digest) self.identifiers.add(identifier) + if self.alternate is not None: + alternate = self.unique_identifier(self.alternate, digest) + self.identifiers.add(alternate) + else: + alternate = None + translated_block = self.dialogue.get(identifier) + if (translated_block is None) and alternate: + translated_block = self.dialogue.get(alternate) if translated_block is None: return block @@ -101,7 +117,11 @@ def translate_dialogue(self, children): if isinstance(i, renpy.ast.Label): if not (hasattr(i, 'hide') and i.hide): - self.label = i.name + if i.name.startswith("_"): + self.alternate = i.name + else: + self.label = i.name + self.alternate = None if self.saving_translations and isinstance(i, renpy.ast.TranslateString) and i.language == self.language: self.strings[i.old] = i.new @@ -110,6 +130,8 @@ def translate_dialogue(self, children): self.walk(i, self.translate_dialogue) elif self.saving_translations and i.language == self.language: self.dialogue[i.identifier] = i.block + if hasattr(i, 'alternate') and i.alternate is not None: + self.dialogue[i.alternate] = i.block if isinstance(i, renpy.ast.Say): group.append(i) From 30b5f45a3a9b22652a53b95237ea3e446476d059 Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Sat, 2 Jan 2021 06:31:19 +0100 Subject: [PATCH 02/18] Add a --try-harder option which attempts to diagnose and fix some common obfuscation tactics. --- README.md | 2 ++ unrpyc.py | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b721a37c..505db2cd 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ Options: insert them. This is always safe to enable if the game's Ren'Py version supports init offset statements, and the generated code is exactly equivalent, only less cluttered. + --try-harder Tries some workarounds against common obfuscation methods. + This is a lot slower. ``` Usage: [python2] unrpyc.py [options] script1 script2 ... diff --git a/unrpyc.py b/unrpyc.py index 2a783e86..8a10ce00 100755 --- a/unrpyc.py +++ b/unrpyc.py @@ -107,8 +107,95 @@ def read_ast_from_file(in_file): data, stmts = magic.safe_loads(raw_contents, class_factory, {"_ast", "collections"}) return stmts +def read_ast_from_file_fuzzy(in_file): + import zlib + import base64 + from collections import Counter + + # In recent times some devs have tried some simple ways of blocking decompilation. This function contains several appraoches + # To defeat this silliness. + raw_contents = in_file.read() + + # Figure out the header offset. Alternatively we could also just straight up try to find the zlib header + position = 0 + while position + 36 < len(raw_contents): + a,b,c,d,e,f,g,h,i = struct.unpack("= 0x20 and ord(i) < 0x80 for i in count.keys()): + raw_contents = raw_contents.decode("string-escape") + print("Encountered a layer of string-escape encoding") + continue + except Exception: + pass + raise Exception("Couldn't figure out the encoding. hint: " + repr("".join(count.keys()))) + else: + raise Exception("Couldn't figure out the encoding") + + return stmts + def decompile_rpyc(input_filename, overwrite=False, dump=False, decompile_python=False, - comparable=False, no_pyexpr=False, translator=None, tag_outside_block=False, init_offset=False): + comparable=False, no_pyexpr=False, translator=None, tag_outside_block=False, + init_offset=False, try_harder=False): # Output filename is input filename but with .rpy extension filepath, ext = path.splitext(input_filename) if dump: @@ -126,7 +213,10 @@ def decompile_rpyc(input_filename, overwrite=False, dump=False, decompile_python return False # Don't stop decompiling if one file already exists with open(input_filename, 'rb') as in_file: - ast = read_ast_from_file(in_file) + if try_harder: + ast = read_ast_from_file_fuzzy(in_file) + else: + ast = read_ast_from_file(in_file) with codecs.open(out_filename, 'w', encoding='utf-8') as out_file: if dump: @@ -134,7 +224,8 @@ def decompile_rpyc(input_filename, overwrite=False, dump=False, decompile_python no_pyexpr=no_pyexpr) else: decompiler.pprint(out_file, ast, decompile_python=decompile_python, printlock=printlock, - translator=translator, tag_outside_block=tag_outside_block, init_offset=init_offset) + translator=translator, tag_outside_block=tag_outside_block, + init_offset=init_offset) return True def extract_translations(input_filename, language): @@ -161,7 +252,8 @@ def worker(t): else: translator = None return decompile_rpyc(filename, args.clobber, args.dump, decompile_python=args.decompile_python, - no_pyexpr=args.no_pyexpr, comparable=args.comparable, translator=translator, tag_outside_block=args.tag_outside_block, init_offset=args.init_offset) + no_pyexpr=args.no_pyexpr, comparable=args.comparable, translator=translator, + tag_outside_block=args.tag_outside_block, init_offset=args.init_offset, try_harder=args.try_harder) except Exception as e: with printlock: print("Error while decompiling %s:" % filename) @@ -221,6 +313,9 @@ def main(): help="The filenames to decompile. " "All .rpyc files in any directories passed or their subdirectories will also be decompiled.") + parser.add_argument('--try-harder', dest="try_harder", action="store_true", + help="Tries some workarounds against common obfuscation methods. This is a lot slower.") + args = parser.parse_args() if args.write_translation_file and not args.clobber and path.exists(args.write_translation_file): From 3cab9f5b15384ead639c7c87fe4bb41057afc8a8 Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Sat, 2 Jan 2021 20:26:50 +0100 Subject: [PATCH 03/18] Make --try-harder cooperate better with multiprocessing --- unrpyc.py | 129 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 67 insertions(+), 62 deletions(-) diff --git a/unrpyc.py b/unrpyc.py index 8a10ce00..5e2fcdbb 100755 --- a/unrpyc.py +++ b/unrpyc.py @@ -126,70 +126,75 @@ def read_ast_from_file_fuzzy(in_file): else: raise Exception("Could not find the header") - with printlock: - if not raw_contents.startswith("RENPY RPC2"): - print("Shenanigans detected, file did not start with default RENPY RPYC2 header") - - if position != 10: - print("Shenanigans detected, header offset was at %s" % position) - - # Normal iteration loop, for now. - chunks = {} - while True: - slot, start, length = struct.unpack("= 0x20 and ord(i) < 0x80 for i in count.keys()): + raw_contents = raw_contents.decode("string-escape") + diagnosis.append("Encountered a layer of string-escape encoding") + continue + except Exception: + pass - # we _assume_ they'd still put the contents in chunk 1 - raw_contents = chunks[1] + # ensure we bail out the following loop as we didn't find anything we recognize + layers = 10 + else: + raise Exception("Couldn't figure out the encoding. tried: \n%s\n, hint: %s" % ("\n".join(diagnosis), repr("".join(count.keys())))) - # In a normal file we're expecting a zlib compressed pickle here, but this is occasionally also changed - layers = 0 - while layers < 10: - layers += 1 - count = Counter(raw_contents) - try: - data, stmts = magic.safe_loads(raw_contents, class_factory, {"_ast", "collections"}) - print("Found the actual pickle") - break - except Exception: - pass - try: - raw_contents = zlib.decompress(raw_contents) - print("Encountered a layer of zlib compression") - continue - except zlib.error: - pass - try: - if all(i in "abcdefABCDEF0123456789" for i in count.keys()): - raw_contents = raw_contents.decode("hex") - print("Encountered a layer of hex encoding") - continue - except TypeError: - pass - try: - # Note: for some reason this doesn't error on characters not part of base64. Just on bad padding - # So it might trigger wrongly - if all(i in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/=\n" for i in count.keys()): - raw_contents = base64.b64decode(raw_contents) - print("Encountered a layer of base64 encoding") - continue - except Exception: - pass - try: - # this is also likely to accept things that aren't actually escaped by it. - if all(ord(i) >= 0x20 and ord(i) < 0x80 for i in count.keys()): - raw_contents = raw_contents.decode("string-escape") - print("Encountered a layer of string-escape encoding") - continue - except Exception: - pass - raise Exception("Couldn't figure out the encoding. hint: " + repr("".join(count.keys()))) - else: - raise Exception("Couldn't figure out the encoding") + with printlock: + for line in diagnosis: + print(line) return stmts From f7e4e186519274a9af0d6e56f68d725917fa68b9 Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Mon, 11 Jan 2021 05:20:09 +0100 Subject: [PATCH 04/18] Move deobfuscation stuff to a separate file. --- deobfuscate.py | 341 +++++++++++++++++++++++++++++++++++++++++++++++++ unrpyc.py | 95 +------------- 2 files changed, 345 insertions(+), 91 deletions(-) create mode 100644 deobfuscate.py diff --git a/deobfuscate.py b/deobfuscate.py new file mode 100644 index 00000000..9e690299 --- /dev/null +++ b/deobfuscate.py @@ -0,0 +1,341 @@ +# Copyright (c) 2021 CensoredUsername +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + + +# This file contains documented strategies used against known obfuscation techniques and some machinery +# to test them against. + +# Architecture is pretty simple. There's at least two steps in unpacking the rpyc format. +# RPYC2 is an archive format that can contain multiple streams (referred to as slots) +# The first step is extracting the slot from it, which is done by one of the extractors. +# These all give a blob that's either still zlib-compressed or just the raw slot +# (some methods rely on the zlib compression to figure out the slot length) +# Then, there's 0 or more steps of decrypting the data in that slot. This ends up often +# being layers of base64, string-escape, hex-encoding, zlib-compression, etc. +# We handle this by just trying these by checking if they fit. + +import os +import zlib +import struct +import base64 +from collections import Counter +from decompiler import magic +import unrpyc + + +# Extractors are simple functions of (fobj, slotno) -> bytes +# They raise ValueError if they fail +EXTRACTORS = [] +def extractor(f): + EXTRACTORS.append(f) + return f + +# Decryptors are simple functions of (bytes, Counter) ->bytes +# They return None if they fail. If they return their input they're also considered to have failed. +DECRYPTORS = [] +def decryptor(f): + DECRYPTORS.append(f) + return f + + +@extractor +def extract_slot_rpyc(f, slot): + """ + Slot extractor for a file that's in the actual rpyc format + """ + f.seek(0) + data = f.read() + if data[:10] != "RENPY RPYC": + raise ValueError("Incorrect Header") + + position = 10 + slots = {} + + while position + 12 <= len(data): + slotid, start, length = struct.unpack("= len(data): + raise ValueError("Broken slot entry") + + slots[slotid] = (start, length) + position += 12 + else: + raise ValueError("Broken slot header structure") + + if slot not in slots: + raise ValueError("Unknown slot id") + + start, length = slots[slot] + return data[start : start + length] + +@extractor +def extract_slot_legacy(f, slot): + """ + Slot extractor for the legacy format + """ + if slot != 1: + raise ValueError("Legacy format only supports 1 slot") + + f.seek(0) + data = f.read() + + try: + data = zlib.decompress(data) + except zlib.error: + raise ValueError("Legacy format did not contain a zlib blob") + + return data + +@extractor +def extract_slot_headerscan(f, slot): + """ + Slot extractor for things that changed the magic and so moved the header around. + """ + f.seek(0) + data = f.read() + + position = 0 + while position + 36 < len(data): + a,b,c,d,e,f,g,h,i = struct.unpack("= len(data): + raise ValueError("Broken slot entry") + + slots[slotid] = (start, length) + position += 12 + else: + raise ValueError("Broken slot header structure") + + if slot not in slots: + raise ValueError("Unknown slot id") + + start, length = slots[slot] + return data[start : start + length] + +@extractor +def extract_slot_zlibscan(f, slot): + """ + Slot extractor for things that fucked with the header structure to the point where it's easier + to just not bother with it and instead we just look for valid zlib chunks directly. + """ + f.seek(0) + data = f.read() + + start_positions = [] + + for i in range(len(data) - 1): + if data[i] != "\x78": + continue + + if (ord(data[i]) * 256 + ord(data[i + 1])) % 31 != 0: + continue + + start_positions.append(i) + + chunks = [] + for position in start_positions: + try: + chunk = zlib.decompress(data[position:]) + except zlib.error: + continue + chunks.append(chunk) + + if slot > len(chunks): + raise ValueError("Zlibscan did not find enough chunks") + + return chunks[slot - 1] + +@decryptor +def decrypt_zlib(data, count): + try: + return zlib.decompress(data) + except zlib.error: + return None + +@decryptor +def decrypt_hex(data, count): + if not all(i in "abcdefABCDEF0123456789" for i in count.keys()): + return None + try: + return data.decode("hex") + except Exception: + return None + +@decryptor +def decrypt_base64(data, count): + if not all(i in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/=\n" for i in count.keys()): + return None + try: + return base64.b64decode(data) + except Exception: + return None + +@decryptor +def decrypt_string_escape(data, count): + if not all(ord(i) >= 0x20 and ord(i) < 0x80 for i in count.keys()): + return None + try: + newdata = data.decode("string-escape") + except Exception: + return None + if newdata == data: + return None + return newdata + + +def assert_is_normal_rpyc(f): + """ + Analyze the structure of a single rpyc file object for correctness. + Does not actually say anything about the _contents_ of that section, just that we were able + to slice it out of there. + + If succesful, returns the uncompressed contents of the first storage slot. + """ + + diagnosis = [] + + f.seek(0) + header = f.read(1024) + f.seek(0) + + if header_data[:10] != "RENPY RPC2": + # either legacy, or someone messed with the header + + # assuming legacy, see if this thing is a valid zlib blob + raw_data = f.read() + f.seek(0) + + try: + uncompressed = zlib.decompress(raw_data) + except zlib.error: + raise ValueError("Did not find RENPY RPC2 header, but interpretation as legacy file failed") + + return uncompressed + + + else: + if len(header) < 46: + # 10 bytes header + 4 * 9 bytes content table + return ValueError("File too short") + + a,b,c,d,e,f,g,h,i = struct.unpack("= 0x20 and ord(i) < 0x80 for i in count.keys()): - raw_contents = raw_contents.decode("string-escape") - diagnosis.append("Encountered a layer of string-escape encoding") - continue - except Exception: - pass - - # ensure we bail out the following loop as we didn't find anything we recognize - layers = 10 - else: - raise Exception("Couldn't figure out the encoding. tried: \n%s\n, hint: %s" % ("\n".join(diagnosis), repr("".join(count.keys())))) - - with printlock: - for line in diagnosis: - print(line) - - return stmts def decompile_rpyc(input_filename, overwrite=False, dump=False, decompile_python=False, comparable=False, no_pyexpr=False, translator=None, tag_outside_block=False, @@ -219,7 +132,7 @@ def decompile_rpyc(input_filename, overwrite=False, dump=False, decompile_python with open(input_filename, 'rb') as in_file: if try_harder: - ast = read_ast_from_file_fuzzy(in_file) + ast = deobfuscate.read_ast(in_file) else: ast = read_ast_from_file(in_file) From 1cc5b33e238c0c6e8a2d45814c9a5fd228f1da74 Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Tue, 12 Jan 2021 20:19:38 +0100 Subject: [PATCH 05/18] textbutton and mousearea were given appropriate styles by ren'py so now we use those to recognize them as well --- decompiler/sl2decompiler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/decompiler/sl2decompiler.py b/decompiler/sl2decompiler.py index 2a21dace..140195db 100644 --- a/decompiler/sl2decompiler.py +++ b/decompiler/sl2decompiler.py @@ -240,6 +240,7 @@ def print_displayable(self, ast, has_block=False): (behavior.OnEvent, None): ("on", 0), (behavior.OnEvent, 0): ("on", 0), (behavior.MouseArea, 0): ("mousearea", 0), + (behavior.MouseArea, None): ("mousearea", 0), (ui._add, None): ("add", 0), (sld.sl2add, None): ("add", 0), (ui._hotbar, "hotbar"): ("hotbar", 0), @@ -247,6 +248,7 @@ def print_displayable(self, ast, has_block=False): (sld.sl2bar, None): ("bar", 0), (ui._label, "label"): ("label", 0), (ui._textbutton, 0): ("textbutton", 0), + (ui._textbutton, "button"): ("textbutton", 0), (ui._imagebutton, "image_button"): ("imagebutton", 0), (im.image, "default"): ("image", 0), (behavior.Input, "input"): ("input", 0), From 681da0f0e65577166e1afe0cde3dfa244324acae Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Tue, 12 Jan 2021 20:38:43 +0100 Subject: [PATCH 06/18] The fillin on unconditional branches changed type from bytes to unicode so now use the presence of a linenumber attribute on it to detect if it was unconditional --- decompiler/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/decompiler/__init__.py b/decompiler/__init__.py index 2667e4b0..045ef838 100644 --- a/decompiler/__init__.py +++ b/decompiler/__init__.py @@ -694,7 +694,10 @@ def print_menu(self, ast): state = None - if isinstance(condition, unicode): + # if the condition is a unicode subclass with a "linenumber" attribute it was script. + # If it isn't ren'py used to insert a "True" string. This string used to be of type str + # but nowadays it's of time unicode, just not of type PyExpr + if isinstance(condition, unicode) and hasattr(condition, "linenumber"): if self.say_inside_menu is not None and condition.linenumber > self.linenumber + 1: # The easy case: we know the line number that the menu item is on, because the condition tells us # So we put the say statement here if there's room for it, or don't if there's not From 3a49cfade57c67332702b65d51084435e1ce1e8b Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Wed, 13 Jan 2021 00:07:21 +0100 Subject: [PATCH 07/18] Also produce a bytecode.rpyb file that does the same as un.rpyc --- un.rpyc/.gitignore | 3 ++- un.rpyc/compile.py | 35 +++++++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/un.rpyc/.gitignore b/un.rpyc/.gitignore index 4261b016..8ee3b7c8 100644 --- a/un.rpyc/.gitignore +++ b/un.rpyc/.gitignore @@ -1,2 +1,3 @@ *.dis* -*.rpyc \ No newline at end of file +*.rpyc +*.rpyb diff --git a/un.rpyc/compile.py b/un.rpyc/compile.py index 16838faf..dce8a27a 100755 --- a/un.rpyc/compile.py +++ b/un.rpyc/compile.py @@ -70,7 +70,7 @@ def Exec(code): pack_folder = path.dirname(path.abspath(__file__)) base_folder = path.dirname(pack_folder) -decompiler = p.ExecTranspile(""" +base = """ # Set up the namespace from os.path import join from os import getcwd @@ -150,7 +150,9 @@ def Exec(code): modules.update(renpy_modules) meta_path.append(renpy_loader) __package__ = package +""" +decompiler_rpyc = p.ExecTranspile(base + """ from renpy import script_version from renpy.game import script ({'version': script_version, 'key': script.key}, []) @@ -165,16 +167,37 @@ def Exec(code): Module("unrpyc", path.join(pack_folder, "unrpyc-compile.py")) )) +decompiler_rpyb = p.ExecTranspile(base + "(None, [])\n", ( + Module("util", path.join(base_folder, "decompiler/util.py")), + Module("magic", path.join(base_folder, "decompiler/magic.py"), False), + Module("codegen", path.join(base_folder, "decompiler/codegen.py")), + Module("testcasedecompiler", path.join(base_folder, "decompiler/testcasedecompiler.py")), + Module("screendecompiler", path.join(base_folder, "decompiler/screendecompiler.py")), + Module("sl2decompiler", path.join(base_folder, "decompiler/sl2decompiler.py")), + Module("decompiler", path.join(base_folder, "decompiler/__init__.py")), + Module("unrpyc", path.join(pack_folder, "unrpyc-compile.py")) +)) + unrpyc = zlib.compress( - p.optimize( - p.dumps(decompiler, protocol), - protocol), - 9) + p.optimize( + p.dumps(decompiler_rpyc, protocol), + protocol), +9) + +bytecoderpyb = zlib.compress( + p.optimize( + p.dumps(decompiler_rpyb, protocol), + protocol), +9) + with open(path.join(pack_folder, "un.rpyc"), "wb") as f: f.write(unrpyc) +with open(path.join(pack_folder, "bytecode.rpyb"), "wb") as f: + f.write(bytecoderpyb) + if args.debug: print("File length = {0}".format(len(unrpyc))) @@ -201,4 +224,4 @@ def Exec(code): pickletools.dis(data, f) with open(path.join(pack_folder, "un.dis3"), "wb" if p.PY2 else "w") as f: - p.pprint(decompiler, f) + p.pprint(decompiler_rpyc, f) From 3f79b213994946ffc38af1168222789dfea0738b Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Wed, 20 Jan 2021 02:52:04 +0100 Subject: [PATCH 08/18] Add a section for people to add custom decryption logic to --- deobfuscate.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deobfuscate.py b/deobfuscate.py index 9e690299..65016f8e 100644 --- a/deobfuscate.py +++ b/deobfuscate.py @@ -56,6 +56,11 @@ def decryptor(f): return f +# Add game-specific custom extraction / decryption logic here + +# End of custom extraction/decryption logic + + @extractor def extract_slot_rpyc(f, slot): """ @@ -177,6 +182,7 @@ def extract_slot_zlibscan(f, slot): return chunks[slot - 1] + @decryptor def decrypt_zlib(data, count): try: From 4e3b67b763e7436b0c58b50df98464e81e8a057a Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Mon, 22 Feb 2021 02:00:23 +0100 Subject: [PATCH 09/18] Support define index expressions --- decompiler/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/decompiler/__init__.py b/decompiler/__init__.py index 045ef838..91f6d424 100644 --- a/decompiler/__init__.py +++ b/decompiler/__init__.py @@ -772,10 +772,13 @@ def print_define(self, ast): init = self.parent if init.priority != self.init_offset and len(init.block) == 1 and not self.should_come_before(init, ast): priority = " %d" % (init.priority - self.init_offset) + index = "" + if hasattr(ast, "index") and ast.index is not None: + index = "[%s]" % ast.index.source if not hasattr(ast, "store") or ast.store == "store": - self.write("%s%s %s = %s" % (name, priority, ast.varname, ast.code.source)) + self.write("%s%s %s%s = %s" % (name, priority, ast.varname, index, ast.code.source)) else: - self.write("%s%s %s.%s = %s" % (name, priority, ast.store[6:], ast.varname, ast.code.source)) + self.write("%s%s %s.%s%s = %s" % (name, priority, ast.store[6:], ast.varname, index, ast.code.source)) # Specials From d5286614b5dddd85bf1c72e24e9cb2ee58e50883 Mon Sep 17 00:00:00 2001 From: madeddy Date: Sat, 23 Jan 2021 21:14:27 +0100 Subject: [PATCH 10/18] Added alternative behavior if multiprocessing module is missing --- unrpyc.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/unrpyc.py b/unrpyc.py index d51b3301..3f7e1739 100755 --- a/unrpyc.py +++ b/unrpyc.py @@ -27,7 +27,13 @@ import itertools import traceback import struct -from multiprocessing import Pool, Lock, cpu_count +try: + from multiprocessing import Pool, Lock, cpu_count +except ImportError: + MP_EXISTS = False + from dummy_thread import allocate_lock +else: + MP_EXISTS = True from operator import itemgetter import decompiler @@ -82,7 +88,7 @@ def __new__(cls, name): class_factory = magic.FakeClassFactory((PyExpr, PyCode, RevertableList, RevertableDict, RevertableSet, Sentinel), magic.FakeStrict) -printlock = Lock() +printlock = Lock() if MP_EXISTS else allocate_lock() # needs class_factory import deobfuscate @@ -184,6 +190,7 @@ def sharelock(lock): def main(): # python27 unrpyc.py [-c] [-d] [--python-screens|--ast-screens|--no-screens] file [file ...] + cc_num = cpu_count() if MP_EXISTS else 1 parser = argparse.ArgumentParser(description="Decompile .rpyc/.rpymc files") parser.add_argument('-c', '--clobber', dest='clobber', action='store_true', @@ -192,8 +199,9 @@ def main(): parser.add_argument('-d', '--dump', dest='dump', action='store_true', help="instead of decompiling, pretty print the ast to a file") - parser.add_argument('-p', '--processes', dest='processes', action='store', default=cpu_count(), - help="use the specified number of processes to decompile") + parser.add_argument('-p', '--processes', dest='processes', action='store', type=int, + choices=range(1, cc_num), default=cc_num - 1 if cc_num > 2 else 1, + help="Used CPU count in multiprocessing, value 1 deactivates") parser.add_argument('-t', '--translation-file', dest='translation_file', action='store', default=None, help="use the specified file to translate during decompilation") @@ -272,7 +280,7 @@ def glob_or_complain(s): files = map(lambda x: (args, x, path.getsize(x)), files) processes = int(args.processes) - if processes > 1: + if MP_EXISTS and processes > 1: # If a big file starts near the end, there could be a long time with # only one thread running, which is inefficient. Avoid this by starting # big files first. From 7d3e42a9213b5939aa847e648a893f77bbf1fb53 Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Sun, 4 Apr 2021 20:42:49 +0200 Subject: [PATCH 11/18] Clean pull request up slightly to remove redundancy and centralize changes --- unrpyc.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/unrpyc.py b/unrpyc.py index 3f7e1739..fe4bdfd2 100755 --- a/unrpyc.py +++ b/unrpyc.py @@ -27,14 +27,24 @@ import itertools import traceback import struct +from operator import itemgetter + try: from multiprocessing import Pool, Lock, cpu_count except ImportError: - MP_EXISTS = False - from dummy_thread import allocate_lock -else: - MP_EXISTS = True -from operator import itemgetter + # Mock required support when multiprocessing is unavailable + def cpu_count(): + return 1 + + class Lock: + def __enter__(self): + pass + def __exit__(self, type, value, traceback): + pass + def acquire(self, block=True, timeout=None): + pass + def release(self): + pass import decompiler from decompiler import magic, astdump, translate @@ -88,7 +98,7 @@ def __new__(cls, name): class_factory = magic.FakeClassFactory((PyExpr, PyCode, RevertableList, RevertableDict, RevertableSet, Sentinel), magic.FakeStrict) -printlock = Lock() if MP_EXISTS else allocate_lock() +printlock = Lock() # needs class_factory import deobfuscate @@ -190,7 +200,7 @@ def sharelock(lock): def main(): # python27 unrpyc.py [-c] [-d] [--python-screens|--ast-screens|--no-screens] file [file ...] - cc_num = cpu_count() if MP_EXISTS else 1 + cc_num = cpu_count() parser = argparse.ArgumentParser(description="Decompile .rpyc/.rpymc files") parser.add_argument('-c', '--clobber', dest='clobber', action='store_true', @@ -201,7 +211,8 @@ def main(): parser.add_argument('-p', '--processes', dest='processes', action='store', type=int, choices=range(1, cc_num), default=cc_num - 1 if cc_num > 2 else 1, - help="Used CPU count in multiprocessing, value 1 deactivates") + help="use the specified number or processes to decompile." + "Defaults to the amount of hw threads available minus one, disabled when muliprocessing is unavailable.") parser.add_argument('-t', '--translation-file', dest='translation_file', action='store', default=None, help="use the specified file to translate during decompilation") @@ -280,7 +291,7 @@ def glob_or_complain(s): files = map(lambda x: (args, x, path.getsize(x)), files) processes = int(args.processes) - if MP_EXISTS and processes > 1: + if processes > 1: # If a big file starts near the end, there could be a long time with # only one thread running, which is inefficient. Avoid this by starting # big files first. From 8fbed82a7870d3079e8b4ed21ea6eece67152242 Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Sun, 4 Apr 2021 20:49:47 +0200 Subject: [PATCH 12/18] Fix #114 --- deobfuscate.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/deobfuscate.py b/deobfuscate.py index 65016f8e..d3f14de0 100644 --- a/deobfuscate.py +++ b/deobfuscate.py @@ -230,13 +230,11 @@ def assert_is_normal_rpyc(f): If succesful, returns the uncompressed contents of the first storage slot. """ - diagnosis = [] - f.seek(0) header = f.read(1024) f.seek(0) - if header_data[:10] != "RENPY RPC2": + if header[:10] != "RENPY RPC2": # either legacy, or someone messed with the header # assuming legacy, see if this thing is a valid zlib blob @@ -256,7 +254,7 @@ def assert_is_normal_rpyc(f): # 10 bytes header + 4 * 9 bytes content table return ValueError("File too short") - a,b,c,d,e,f,g,h,i = struct.unpack(" Date: Sun, 4 Apr 2021 21:42:27 +0200 Subject: [PATCH 13/18] Update readme --- README.md | 197 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 144 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 505db2cd..681e527e 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,91 @@ -master: [![Build Status](https://travis-ci.org/CensoredUsername/unrpyc.svg?branch=master)](https://travis-ci.org/CensoredUsername/unrpyc) - -dev: [![Build Status](https://travis-ci.org/CensoredUsername/unrpyc.svg?branch=dev)](https://travis-ci.org/CensoredUsername/unrpyc) +# Unrpyc, the Ren'py script decompiler. -Unrpyc is a script to decompile Ren'Py (http://www.renpy.org/) compiled .rpyc +Unrpyc is a tool to decompile Ren'Py (http://www.renpy.org/) compiled .rpyc script files. It will not extract files from .rpa archives. For that, use [rpatool](https://github.com/Shizmob/rpatool) or [UnRPA](https://github.com/Lattyware/unrpa). -Thanks to recent changes, unrpyc no longer needs internal renpy structures to -work. -Usage options: +## Status -Options: -``` - --version show program's version number and exit +master: [![Build Status](https://travis-ci.org/CensoredUsername/unrpyc.svg?branch=master)](https://travis-ci.org/CensoredUsername/unrpyc) + +dev: [![Build Status](https://travis-ci.org/CensoredUsername/unrpyc.svg?branch=dev)](https://travis-ci.org/CensoredUsername/unrpyc) + +## Usage - -h, --help show this help message and exit +This tool can either be ran as a command line tool, as a library, or injected into the game itself. It requires Python 2.7 to be installed to be used as a command line tool. - -c, --clobber overwrites existing output files +### Command line tool usage - -d, --dump Instead of decompiling, pretty print the contents - of the AST in a human readable format. - This is mainly useful for debugging. +Depending on your system setup, you should use one of the following commands to run the tool: +``` +python unrpyc.py [options] script1 script2 ... +python2 unrpyc.py [options] script1 script2 ... +py -2 unrpyc.py [options] script1 script2 ... +./unrpyc.py [options] script1 script2 ... +``` + +Options: +``` +$ py -2 unrpyc.py --help +usage: unrpyc.py [-h] [-c] [-d] [-p {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}] + [-t TRANSLATION_FILE] [-T WRITE_TRANSLATION_FILE] + [-l LANGUAGE] [--sl1-as-python] [--comparable] [--no-pyexpr] + [--tag-outside-block] [--init-offset] [--try-harder] + file [file ...] + +Decompile .rpyc/.rpymc files + +positional arguments: + file The filenames to decompile. All .rpyc files in any + directories passed or their subdirectories will also + be decompiled. + +optional arguments: + -h, --help show this help message and exit + -c, --clobber overwrites existing output files + -d, --dump instead of decompiling, pretty print the ast to a file -p, --processes - use the specified number of processes to decompile - --sl1-as-python - Only dumping and for decompiling screen language 1 - screens. Convert SL1 Python AST to Python code instead - of dumping it or converting it to screenlang. - --comparable Only for dumping, remove several false differences when - comparing dumps. This suppresses attributes that are - different even when the code is identical, such as file - modification times. - --no-pyexpr Only for dumping, disable special handling of PyExpr objects, - instead printing them as strings. This is useful when comparing - dumps from different versions of Ren'Py. It should only be used - if necessary, since it will cause loss of information such as - line numbers. - --init-offset Attempt to guess when init offset statements were used and - insert them. This is always safe to enable if the game's Ren'Py - version supports init offset statements, and the generated code - is exactly equivalent, only less cluttered. - --try-harder Tries some workarounds against common obfuscation methods. - This is a lot slower. + use the specified number or processes to + decompile.Defaults to the amount of hw threads + available minus one, disabled when muliprocessing is + unavailable. + -t TRANSLATION_FILE, --translation-file TRANSLATION_FILE + use the specified file to translate during + decompilation + -T WRITE_TRANSLATION_FILE, --write-translation-file WRITE_TRANSLATION_FILE + store translations in the specified file instead of + decompiling + -l LANGUAGE, --language LANGUAGE + if writing a translation file, the language of the + translations to write + --sl1-as-python Only dumping and for decompiling screen language 1 + screens. Convert SL1 Python AST to Python code instead + of dumping it or converting it to screenlang. + --comparable Only for dumping, remove several false differences + when comparing dumps. This suppresses attributes that + are different even when the code is identical, such as + file modification times. + --no-pyexpr Only for dumping, disable special handling of PyExpr + objects, instead printing them as strings. This is + useful when comparing dumps from different versions of + Ren'Py. It should only be used if necessary, since it + will cause loss of information such as line numbers. + --tag-outside-block Always put SL2 'tag's on the same line as 'screen' + rather than inside the block. This will break + compiling with Ren'Py 7.3 and above, but is needed to + get correct line numbers from some files compiled with + older Ren'Py versions. + --init-offset Attempt to guess when init offset statements were used + and insert them. This is always safe to enable if the + game's Ren'Py version supports init offset statements, + and the generated code is exactly equivalent, only + less cluttered. + --try-harder Tries some workarounds against common obfuscation + methods. This is a lot slower. + ``` -Usage: [python2] unrpyc.py [options] script1 script2 ... You can give several .rpyc files on the command line. Each script will be decompiled to a corresponding .rpy on the same directory. Additionally, you can @@ -62,29 +102,80 @@ open an issue to alert us of the problem. For the script to run correctly it is required for the unrpyc.py file to be in the same directory as the modules directory. -You can also import the module from python and call -unrpyc.decompile_rpyc(filename, ...) directly +### Game injection + +The tool can be injected directly into a running game by placing either the +`un.rpyc` file or the `bytecode.rpyb` file from the most recent release into +the `game` directory inside a Ren'py game. When the game is then ran the tool +will automatically extract and decompile all game script files into the `game` +directory. The tool writes logs to the file `unrpyc.log.txt`. + +### Library usage + +You can import the module from python and call +unrpyc.decompile_rpyc(filename, ...) directly. -As of renpy version 6.18 the way renpy handles screen language changed -significantly. Due to this significant changes had to be made, and the script -might be less stable for older renpy versions. If you encounter any problems -due to this, please report them. +## Notes on support -Alternatively there is an experimental version of the decompiler packed into -one file available at https://github.com/CensoredUsername/unrpyc/releases -This version will decompile a game from inside the renpy runtime. Simply copy -the un.rpyc file into the "game" directory inside the game files and everything -will be decompiled. +The Ren'py engine has changed a lot through the years. While this tool tries to +support all available Ren'py versions since the creation of this tool, we do not +actively test it against every engine release. Furthermore the engine does +not have perfect backwards compatibility itself, so issues can occur if you try +to run decompiled files with different engine releases. Most attention is given +to recent engine versions so if you encounter an issues with older games, please +report it. Supported: -* renpy version 6 +* renpy version 6 and 7 (current) * Windows, OSX and Linux -Unrpyc has only been tested on versions up to 6.99.9, though newer versions are -expected to mostly work. If you find an error due to a new ren'py version being -incompatible, please open an issue. +## Issue reports + +As Ren'py is being continuously developed itself it often occurs that this tool might +break on newer engine releases. This is most likely due to us not being +aware of these features existing in the first place. To get this fixed +you can make an issue report to this repository. However, we work on this tool +in our free time and therefore we strongly request performing the following steps when +making an issue report. + +### Before making an issue report: + +If you are making an issue report because decompilation errors out, please do the following. +If there's simply an error in the decompiled file, you can skip these steps. + +1. Test your .rpyc files with the command line tool and both game injection methods. Please + do this directly, do not use wrapper tools incorporating unrpyc for the report. +2. Run the command line tool with the anti-obfuscation option `--try-harder`. + +### When making an issue report: + +1. List the used version of unrpyc and the version of ren'py used to create the .rpyc file + you're trying to decompile (and if applicable, what game). +2. Describe exactly what you're trying to do, and what the issue is (is it not decompiling + at all, is there an omission in the decompiled file, or is the decompiled file invalid). +3. Attach any relevant output produced by the tool (full command line output is preferred, + if output is generated attach that as well). +4. Attach the .rpyc file that is failing to decompile properly. + +Please perform all these steps, and write your issue report in legible English. Otherwise +it is likely that your issue report will just receive a reminder to follow these steps. + +## Feature and pull requests + +Feature and pull requests are welcome. Feature requests will be handled whenever we feel +like it, so if you really want a feature in the tool a pull request is usually the right +way to go. Please do your best to conform to the style used by the rest of the code base +and only affect what's absolutely necessary, this keeps the process smooth. + +### Notes on deobfuscation -Requirements: -* Python version 2.7 +Recently a lot of modifications of Ren'py have turned up that slightly alter the Ren'py +file format to block this tool from working. The tool now includes a basic framework +for deobfuscation, but feature requests to create deobfuscation support for specific +games are not likely to get a response from us as this is essentially just an arms race, +and it's trivial to figure out a way to obfuscate the file that blocks anything that is +supported right now. If you make a pull request with it we'll happily put it in mainline +or a game-specific branch depending on how many games it affects, but we have little +motivation ourselves to put time in this arms race. https://github.com/CensoredUsername/unrpyc From 80e4b8b6ac72f5cc55ec05a5b2c53b2392e6615f Mon Sep 17 00:00:00 2001 From: madeddy Date: Fri, 9 Apr 2021 07:52:58 +0200 Subject: [PATCH 14/18] Fix wrong string and type in file magic comparison --- deobfuscate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deobfuscate.py b/deobfuscate.py index d3f14de0..a83e7d3e 100644 --- a/deobfuscate.py +++ b/deobfuscate.py @@ -68,7 +68,7 @@ def extract_slot_rpyc(f, slot): """ f.seek(0) data = f.read() - if data[:10] != "RENPY RPYC": + if data[:10] != b'RENPY RPC2': raise ValueError("Incorrect Header") position = 10 @@ -234,7 +234,7 @@ def assert_is_normal_rpyc(f): header = f.read(1024) f.seek(0) - if header[:10] != "RENPY RPC2": + if header[:10] != b'RENPY RPC2': # either legacy, or someone messed with the header # assuming legacy, see if this thing is a valid zlib blob From 0989b65acd0ebb4f731b8d8a22e873ad041724af Mon Sep 17 00:00:00 2001 From: Asriel Senna Date: Sat, 24 Apr 2021 16:41:07 +0200 Subject: [PATCH 15/18] add LanguageCases in the same way as _ELSE_COND (may be simplified) --- unrpyc.py | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/unrpyc.py b/unrpyc.py index 48dd3a20..96ecac8d 100755 --- a/unrpyc.py +++ b/unrpyc.py @@ -108,7 +108,46 @@ def __new__(cls, s): cls.instance = str.__new__(cls, s) return cls.instance -class_factory = magic.FakeClassFactory((PyExpr, PyCode, RevertableList, RevertableDict, RevertableSet, Sentinel, RevertableList, _ELSE_COND), magic.FakeStrict) +class LanguageCases(magic.FakeStrict, unicode): + __module__ = "store" + __slots__ = [ + "_cases", + ] + def __new__(cls, s, **cases): + self = unicode.__new__(cls, s) + self._cases = dict(cases) + return self + + def __getstate__(self): + return self._cases + + def __setstate__(self, cases): + self._cases.update(cases) + + def __getattr__(self, name): + if name.startswith("__") and name.endswith("__"): + raise AttributeError() + + if name in self._cases: + return self._cases[name] + + language = preferences.language + if language is None: + rv = None + language = "english" + else: + rv = self._cases.get(language + "_" + name) + + if rv is not None: + return rv + elif config.developer: + raise AttributeError( + "LanguageCases have not defined " + "'{}' case for language '{}'".format(name, language)) + else: + return __(self) + +class_factory = magic.FakeClassFactory((PyExpr, PyCode, RevertableList, RevertableDict, RevertableSet, Sentinel, RevertableList, _ELSE_COND, LanguageCases), magic.FakeStrict) printlock = Lock() From 5819ecef03531ac1356671e583a672d08c96c4da Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Sat, 8 May 2021 00:12:39 +0200 Subject: [PATCH 16/18] Fix #121. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7b7ba54b..b6de838b 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,6 @@ def readme(): url='https://github.com/CensoredUsername/unrpyc', py_modules=['unrpyc'], packages=['decompiler'], - scripts=['unrpyc.py'], + scripts=['unrpyc.py', 'deobfuscate.py'], zip_safe=False, ) From 77e01d16fdabae46f30a60ab21b85b4d4148fb26 Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Mon, 17 May 2021 00:05:03 +0200 Subject: [PATCH 17/18] Handle broken sl2 keywords properly --- decompiler/sl2decompiler.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/decompiler/sl2decompiler.py b/decompiler/sl2decompiler.py index 140195db..54e78c23 100644 --- a/decompiler/sl2decompiler.py +++ b/decompiler/sl2decompiler.py @@ -294,21 +294,46 @@ def print_keywords_and_children(self, keywords, children, lineno, needs_colon=Fa keywords_somewhere.extend(("tag", tag)) else: current_line[1].extend(("tag", tag)) + + force_newline = False for key, value in keywords: if value is None: - value = "" - if current_line[0] is None: + # ok, so normally this wouldn't make sense to be None, it should be a PyExpr. However + # ren'py's parser is broken and instead of erroring on a keyword argument that hasn't got + # an actual value given it instead just inserts `None` as the value. Since basically every keyword + # is technically a valid expression the only way for this to happen is at the end of a line, + # so after this we have to force a line break + + # if this is the first keyword, or the previous was broken we need to force a newline + if current_line[0] is None or force_newline: + force_newline = False keywords_by_line.append(current_line) current_line = (0, []) - elif current_line[0] is None or value.linenumber > current_line[0]: - keywords_by_line.append(current_line) - current_line = (value.linenumber, []) - current_line[1].extend((key, value)) + + # force a newline + force_newline = True + + # just output the key + current_line[1].append(key) + + else: + if current_line[0] is None or value.linenumber > current_line[0] or force_newline: + force_newline = False + keywords_by_line.append(current_line) + current_line = (value.linenumber, []) + + current_line[1].extend((key, value)) + if keywords_by_line: + if force_newline: + keywords_by_line.append(current_line) + current_line = (0, []) + # Easy case: we have at least one line inside the block that already has keywords. # Just put the ones from keywords_somewhere with them. current_line[1].extend(keywords_somewhere) keywords_somewhere = [] + keywords_by_line.append(current_line) last_keyword_line = keywords_by_line[-1][0] children_with_keywords = [] From 62b14f8a52ac441643e42e3435d88e88cebefdcb Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Fri, 19 Nov 2021 17:41:21 +0100 Subject: [PATCH 18/18] use game's own rpyc loading function inside un.rpyc and add un.rpy to prevent rpyc modification shenanigans. --- un.rpyc/compile.py | 85 +++++++++++++++++++++++++++++++++++++-- un.rpyc/unrpyc-compile.py | 15 ------- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/un.rpyc/compile.py b/un.rpyc/compile.py index dce8a27a..a8724eb3 100755 --- a/un.rpyc/compile.py +++ b/un.rpyc/compile.py @@ -24,6 +24,7 @@ import argparse import os, sys import minimize +import base64 from os import path parser = argparse.ArgumentParser(description="Pack unpryc into un.rpyc which can be ran from inside renpy") @@ -52,13 +53,13 @@ except ImportError: exit("Could not import pickleast. Are you sure it's in pythons module search path?") -def Module(name, filename, munge_globals=True): +def Module(name, filename, munge_globals=True, retval=False): with open(filename, "rb" if p.PY2 else "r") as f: code = f.read() if args.minimize: # in modules only locals are worth optimizing code = minimize.minimize(code, True, args.obfuscate and munge_globals, args.obfuscate, args.obfuscate) - return p.Module(name, code) + return p.Module(name, code, retval=retval) def Exec(code): if args.minimize: @@ -95,7 +96,7 @@ def Exec(code): files = listdirfiles() exec ''' -import os, sys, renpy +import os, sys, renpy, zlib sys.init_offset = renpy.version_tuple >= (6, 99, 10, 1224) sys.files = [] for (dir, fn) in files: @@ -107,7 +108,8 @@ def Exec(code): elif (dir, fn[:-1]) not in files: abspath = os.path.join(dir, fn) if dir else os.path.join(basepath, fn) with renpy.loader.load(fn) as file: - sys.files.append((abspath, fn, dir, file.read())) + bindata = renpy.game.script.read_rpyc_data(file, 1) + sys.files.append((abspath, fn, dir, bindata)) ''' in globals() _0 # util @@ -178,6 +180,73 @@ def Exec(code): Module("unrpyc", path.join(pack_folder, "unrpyc-compile.py")) )) +rpy_one = p.GetItem(p.Sequence( + Module("util", path.join(base_folder, "decompiler/util.py")), + Module("magic", path.join(base_folder, "decompiler/magic.py"), False), + Module("codegen", path.join(base_folder, "decompiler/codegen.py")), +), "magic") + +rpy_two = p.GetItem(p.Sequence( + Module("testcasedecompiler", path.join(base_folder, "decompiler/testcasedecompiler.py")), + Module("screendecompiler", path.join(base_folder, "decompiler/screendecompiler.py")), + Module("sl2decompiler", path.join(base_folder, "decompiler/sl2decompiler.py")), + Module("decompiler", path.join(base_folder, "decompiler/__init__.py")), + Module("unrpyc", path.join(pack_folder, "unrpyc-compile.py")) +), "unrpyc") + +rpy_base = """ +init python early hide: + + # Set up the namespace + import os + import os.path + import sys + import renpy + import renpy.loader + import base64 + import pickle + import zlib + + basepath = os.path.join(os.getcwd(), "game") + files = renpy.loader.listdirfiles() + + sys.init_offset = renpy.version_tuple >= (6, 99, 10, 1224) + sys.files = [] + for (dir, fn) in files: + if fn.endswith((".rpyc", ".rpymc")): + if dir and dir.endswith("common"): + continue + elif fn == "un.rpyc": + continue + elif (dir, fn[:-1]) not in files: + abspath = os.path.join(dir, fn) if dir else os.path.join(basepath, fn) + with renpy.loader.load(fn) as file: + bindata = renpy.game.script.read_rpyc_data(file, 1) + sys.files.append((abspath, fn, dir, bindata)) + + # ??? + magic = pickle.loads(base64.b64decode({})) + + renpy_modules = sys.modules.copy() + for i in renpy_modules: + if b"renpy" in i and not b"renpy.execution" in i: + sys.modules.pop(i) + + renpy_loader = sys.meta_path.pop() + + magic.fake_package(b"renpy") + magic.FakeModule(b"astdump") + magic.FakeModule(b"translate") + + # ????????? + unrpyc = pickle.loads(base64.b64decode({})) + unrpyc.decompile_game() + + magic.remove_fake_package(b"renpy") + + sys.modules.update(renpy_modules) + sys.meta_path.append(renpy_loader) +""" unrpyc = zlib.compress( p.optimize( @@ -191,6 +260,11 @@ def Exec(code): protocol), 9) +unrpy = rpy_base.format( + repr(base64.b64encode(p.optimize(p.dumps(rpy_one, protocol), protocol))), + repr(base64.b64encode(p.optimize(p.dumps(rpy_two, protocol), protocol))) +) + with open(path.join(pack_folder, "un.rpyc"), "wb") as f: f.write(unrpyc) @@ -198,6 +272,9 @@ def Exec(code): with open(path.join(pack_folder, "bytecode.rpyb"), "wb") as f: f.write(bytecoderpyb) +with open(path.join(pack_folder, "un.rpy"), "wb") as f: + f.write(unrpy) + if args.debug: print("File length = {0}".format(len(unrpyc))) diff --git a/un.rpyc/unrpyc-compile.py b/un.rpyc/unrpyc-compile.py index 102c70b3..0714d460 100644 --- a/un.rpyc/unrpyc-compile.py +++ b/un.rpyc/unrpyc-compile.py @@ -74,21 +74,6 @@ def __new__(cls, name): factory = magic.FakeClassFactory((PyExpr, PyCode, RevertableList, RevertableDict, RevertableSet, Sentinel), magic.FakeStrict) def read_ast_from_file(raw_contents): - # .rpyc files are just zlib compressed pickles of a tuple of some data and the actual AST of the file - if raw_contents.startswith("RENPY RPC2"): - # parse the archive structure - position = 10 - chunks = {} - while True: - slot, start, length = struct.unpack("III", raw_contents[position: position + 12]) - if slot == 0: - break - position += 12 - - chunks[slot] = raw_contents[start: start + length] - - raw_contents = chunks[1] - raw_contents = raw_contents.decode('zlib') data, stmts = magic.safe_loads(raw_contents, factory, {"_ast", "collections"}) return stmts