diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4e47a0 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/util/__pycache__ diff --git a/requirements.txt b/requirements.txt index be753ae..1b03a05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ datetime mistune arpy Pillow -pydpkg \ No newline at end of file +pydpkg +pyzstd \ No newline at end of file diff --git a/util/DebianPackager.py b/util/DebianPackager.py index 3dc6e5d..fcb38ec 100644 --- a/util/DebianPackager.py +++ b/util/DebianPackager.py @@ -33,9 +33,11 @@ def CompileRelease(self, repo_settings): release_file += "Suite: stable\n" release_file += "Version: 1.0\n" release_file += "Codename: ios\n" - release_file += "Architectures: iphoneos-arm\n" + release_file += "Architectures: iphoneos-arm iphoneos-arm64\n" release_file += "Components: main\n" - release_file += "Description: " + repo_settings['description'].replace("\n\n", "\n .\n ").replace("\n", "\n ") + "\n" + release_file += "Description: " + \ + repo_settings['description'].replace( + "\n\n", "\n .\n ").replace("\n", "\n ") + "\n" return release_file @@ -47,8 +49,7 @@ def CompileControl(self, tweak_data, repo_settings): Object repo_settings: An object of repo settings. """ subfolder = PackageLister.FullPathCname(self, repo_settings) - - control_file = "Architecture: iphoneos-arm\n" + control_file = "Architecture: iphoneos-arm64\n" # Mandatory properties include name, bundle id, and version. control_file += "Package: " + tweak_data['bundle_id'] + "\n" control_file += "Name: " + tweak_data['name'] + "\n" @@ -56,7 +57,7 @@ def CompileControl(self, tweak_data, repo_settings): # Known properties control_file += "Depiction: https://" + repo_settings['cname'] + subfolder + "/depiction/web/" + tweak_data[ 'bundle_id'] \ - + ".html\n" + + ".html\n" control_file += "SileoDepiction: https://" + repo_settings['cname'] + subfolder + "/depiction/native/" \ + tweak_data['bundle_id'] + ".json\n" control_file += "ModernDepiction: https://" + repo_settings['cname'] + subfolder + "/depiction/native/" \ @@ -68,7 +69,9 @@ def CompileControl(self, tweak_data, repo_settings): try: if tweak_data['tagline']: # APT note: Multi-line descriptions are in the spec, but must be indicated with a leading space. - control_file += "Description: " + tweak_data['tagline'].replace("\n\n", "\n .\n ").replace("\n", "\n ") + "\n" + control_file += "Description: " + \ + tweak_data['tagline'].replace( + "\n\n", "\n .\n ").replace("\n", "\n ") + "\n" except Exception: control_file += "Description: An awesome package!\n" @@ -86,7 +89,8 @@ def CompileControl(self, tweak_data, repo_settings): try: if tweak_data['pre_dependencies']: - control_file += "Pre-Depends: " + tweak_data['pre_dependencies'] + "\n" + control_file += "Pre-Depends: " + \ + tweak_data['pre_dependencies'] + "\n" except Exception: pass @@ -95,7 +99,8 @@ def CompileControl(self, tweak_data, repo_settings): control_file += "Depends: firmware (>=" + tweak_data['works_min'] + "), " + tweak_data[ 'dependencies'] + "\n" except Exception: - control_file += "Depends: firmware (>=" + tweak_data['works_min'] + ")\n" + control_file += "Depends: firmware (>=" + \ + tweak_data['works_min'] + ")\n" try: if tweak_data['conflicts']: @@ -117,13 +122,15 @@ def CompileControl(self, tweak_data, repo_settings): try: if tweak_data['build_depends']: - control_file += "Build-Depends: " + tweak_data['build_depends'] + "\n" + control_file += "Build-Depends: " + \ + tweak_data['build_depends'] + "\n" except Exception: pass try: if tweak_data['recommends']: - control_file += "Recommends: " + tweak_data['recommends'] + "\n" + control_file += "Recommends: " + \ + tweak_data['recommends'] + "\n" except Exception: pass @@ -146,9 +153,13 @@ def CompileControl(self, tweak_data, repo_settings): pass try: if tweak_data['tags']: - control_file += "Tags: compatible_min::ios" + tweak_data['works_min'] + ", compatible_max::ios" + tweak_data['works_max'] + ", " + tweak_data['tags'] + "\n" + control_file += "Tags: compatible_min::ios" + \ + tweak_data['works_min'] + ", compatible_max::ios" + \ + tweak_data['works_max'] + ", " + tweak_data['tags'] + "\n" except Exception: - control_file += "Tags: compatible_min::ios" + tweak_data['works_min'] + ", compatible_max::ios" + tweak_data['works_max'] + "\n" + control_file += "Tags: compatible_min::ios" + \ + tweak_data['works_min'] + ", compatible_max::ios" + \ + tweak_data['works_max'] + "\n" try: if tweak_data['developer']: @@ -157,7 +168,8 @@ def CompileControl(self, tweak_data, repo_settings): control_file += "Author: " + tweak_data['developer']['name'] + " <" + tweak_data['developer'][ 'email'] + ">\n" except Exception: - control_file += "Author: " + tweak_data['developer']['name'] + "\n" + control_file += "Author: " + \ + tweak_data['developer']['name'] + "\n" except Exception: control_file += "Author: Unknown\n" @@ -167,7 +179,8 @@ def CompileControl(self, tweak_data, repo_settings): + tweak_data['maintainer']['email'] + ">\n" except Exception: try: - control_file += "Maintainer: " + tweak_data['maintainer']['name'] + "\n" + control_file += "Maintainer: " + \ + tweak_data['maintainer']['name'] + "\n" except Exception: try: if tweak_data['developer']['email']: @@ -175,7 +188,8 @@ def CompileControl(self, tweak_data, repo_settings): + tweak_data['developer']['email'] + ">\n" except Exception: try: - control_file += "Maintainer: " + tweak_data['developer']['name'] + "\n" + control_file += "Maintainer: " + \ + tweak_data['developer']['name'] + "\n" except Exception: control_file += "Maintainer: Unknown\n" @@ -186,7 +200,8 @@ def CompileControl(self, tweak_data, repo_settings): control_file += "Sponsor: " + tweak_data['sponsor']['name'] + " <" + tweak_data['sponsor'][ 'email'] + ">\n" except Exception: - control_file += "Sponsor: " + tweak_data['sponsor']['name'] + "\n" + control_file += "Sponsor: " + \ + tweak_data['sponsor']['name'] + "\n" except Exception: pass @@ -217,7 +232,8 @@ def CreateDEB(self, bundle_id, recorded_version): deb = Dpkg(self.root + "temp/" + bundle_id + "/" + file_name) if Dpkg.compare_versions(recorded_version, deb.version) == -1: # Update package stuff - package_name = PackageLister.BundleIdToDirName(self, bundle_id) + package_name = PackageLister.BundleIdToDirName( + self, bundle_id) with open(self.root + "Packages/" + package_name + "/silica_data/index.json", "r") as content_file: update_json = json.load(content_file) update_json['version'] = deb.version @@ -250,31 +266,40 @@ def CreateDEB(self, bundle_id, recorded_version): # tweak_release variable again, which wrecks a lot of things (ie runtime). # A Silica rewrite is required to properly fix this bug. print("\nA small warning about adding changelogs mid-run:\n") - print("Due to some less-than-ideal design decisions with Silica, for the changelog to show") - print("up, you're going to have to run Silica again. Yes, I know this is annoying, and a proper") - print("solution is in the works, but the under-the-hood changes that'll be needed to fix") - print("it properly would require a rewrite [see issue #22].\n") + print( + "Due to some less-than-ideal design decisions with Silica, for the changelog to show") + print( + "up, you're going to have to run Silica again. Yes, I know this is annoying, and a proper") + print( + "solution is in the works, but the under-the-hood changes that'll be needed to fix") + print( + "it properly would require a rewrite [see issue #22].\n") print("I'm deeply sorry about this.\n - Shuga.\n") # Get human-readable folder name - folder = PackageLister.BundleIdToDirName(self, bundle_id) + folder = PackageLister.BundleIdToDirName( + self, bundle_id) deb_path = self.root + "Packages/" + folder + "/" + file_name # Extract Control file and scripts from DEB DpkgPy.control_extract(self, deb_path, self.root + "Packages/" + folder + "/silica_data/scripts/") # Remove the Control; it's not needed. - os.remove(self.root + "Packages/" + folder + "/silica_data/scripts/control") + os.remove(self.root + "Packages/" + folder + + "/silica_data/scripts/control") if not os.listdir(self.root + "Packages/" + folder + "/silica_data/scripts/"): - os.rmdir(self.root + "Packages/" + folder + "/silica_data/scripts/") + os.rmdir(self.root + "Packages/" + + folder + "/silica_data/scripts/") return_str = json.dumps(update_json) print("Updating package index.json...") PackageLister.CreateFile(self, "Packages/" + package_name + "/silica_data/index.json", return_str) pass - DpkgPy.extract(self, self.root + "temp/" + bundle_id + "/" + file_name, self.root + "temp/" + bundle_id) + DpkgPy.extract(self, self.root + "temp/" + bundle_id + + "/" + file_name, self.root + "temp/" + bundle_id) try: - os.remove(self.root + "temp/" + bundle_id + "/" + file_name) + os.remove(self.root + "temp/" + + bundle_id + "/" + file_name) except: pass try: @@ -288,23 +313,26 @@ def CreateDEB(self, bundle_id, recorded_version): # Check for a DEB that already exists. docs_deb = Dpkg(self.root + "docs/pkg/" + bundle_id + ".deb") if docs_deb.version == recorded_version: - shutil.copy(self.root + "docs/pkg/" + bundle_id + ".deb", self.root + "temp/" + bundle_id + ".deb") - call_result = 0; + shutil.copy(self.root + "docs/pkg/" + bundle_id + + ".deb", self.root + "temp/" + bundle_id + ".deb") + call_result = 0 else: # Sneaky swap. - call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB + call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB except: # Create the DEB again. - call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB + call_result = call(["dpkg-deb", "-b", "-Zgzip", self.root + + "temp/" + bundle_id], cwd=self.root + "temp/") # Compile DEB if call_result != 0: # Did we run within WSL? if "Microsoft" in platform.release(): PackageLister.ErrorReporter(self, "Platform Error!", "dpkg-deb failed to run. " - "This is likely due to improper configuration of WSL. Please check the Silcia README for " - "how to set up WSL for dpkg-deb.") + "This is likely due to improper configuration of WSL. Please check the Silcia README for " + "how to set up WSL for dpkg-deb.") else: PackageLister.ErrorReporter(self, "DPKG Error!", "dpkg-deb failed to run. " - "This could be due to a faulty system configuration.") + "This could be due to a faulty system configuration.") def CheckForSilicaData(self): """ @@ -317,7 +345,8 @@ def CheckForSilicaData(self): for folder in os.listdir(self.root + "Packages"): if folder.lower() != ".ds_store": if not os.path.isdir(self.root + "Packages/" + folder + "/silica_data"): - print("It seems like the package \"" + folder + "\" is not configured. Let's set it up!") + print("It seems like the package \"" + folder + + "\" is not configured. Let's set it up!") is_deb = False deb_path = "" try: @@ -327,13 +356,14 @@ def CheckForSilicaData(self): deb_path = self.root + "Packages/" + folder + "/" + file_name except Exception: PackageLister.ErrorReporter(self, "Configuration Error!", "Please put your .deb file inside of " - "its own folder. The \"Packages\" directory should be made of multiple folders that each " - "contain data for a single package.\n Please fix this issue and try again.") + "its own folder. The \"Packages\" directory should be made of multiple folders that each " + "contain data for a single package.\n Please fix this issue and try again.") # This will be the default scaffolding for our package. Eventually I'll neuter it to only be the # essential elements; it's also kinda a reference to me. output = { "bundle_id": "co.shuga.silica.unknown", + "Architecture": "iphoneos-arm", "name": "Unknown Package", "version": "1.0.0", "tagline": "An unknown package.", @@ -358,30 +388,41 @@ def CheckForSilicaData(self): deb = Dpkg(deb_path) output['name'] = deb.headers['Name'] output['bundle_id'] = deb.headers['Package'] + try: + output['Architecture'] = deb.headers['Architecture'] + except Exception: + output['Architecture'] = input( + "What is the Architecture(iphoneos-arm) of the package? ") try: output['tagline'] = deb.headers['Description'] except Exception: - output['tagline'] = input("What is a brief description of the package? ") + output['tagline'] = input( + "What is a brief description of the package? ") try: output['homepage'] = deb.headers['Homepage'] except Exception: pass try: remove_email_regex = re.compile('<.*?>') - output['developer']['name'] = remove_email_regex.sub("", deb.headers['Author']) + output['developer']['name'] = remove_email_regex.sub( + "", deb.headers['Author']) except Exception: output['developer']['name'] = input("Who originally made this package? This may be" " your name. ") - output['developer']['email'] = input("What is the original author's email address? ") + output['developer']['email'] = input( + "What is the original author's email address? ") try: remove_email_regex = re.compile('<.*?>') - output['maintainer']['name'] = remove_email_regex.sub("", deb.headers['Maintainer']) + output['maintainer']['name'] = remove_email_regex.sub( + "", deb.headers['Maintainer']) except Exception: output['maintainer']['name'] = input("Who maintains this package now?" " This is likely your name. ") - output['maintainer']['email'] = input("What is the maintainer's email address? ") + output['maintainer']['email'] = input( + "What is the maintainer's email address? ") try: - output['sponsor']['name'] = remove_email_regex.sub("", deb.headers['Sponsor']) + output['sponsor']['name'] = remove_email_regex.sub( + "", deb.headers['Sponsor']) except Exception: pass try: @@ -437,9 +478,16 @@ def CheckForSilicaData(self): except Exception: pass # These still need data. - output['works_min'] = input("What is the lowest iOS version the package works on? ") - output['works_max'] = input("What is the highest iOS version the package works on? ") - output['featured'] = input("Should this package be featured on your repo? (true/false) ") + output['works_min'] = input( + "What is the lowest iOS version the package works on? ") + + if output['works_min'] == "": + output['works_min'] = "15.0" + + output['works_max'] = input( + "What is the highest iOS version the package works on? ") + output['featured'] = input( + "Should this package be featured on your repo? (true/false) ") set_tint = input("What would you like this package's tint color to be? To keep it at" " the default, leave this blank: ") if set_tint != "": @@ -447,12 +495,15 @@ def CheckForSilicaData(self): print("All done! Please look over the generated \"index.json\" file and consider populating the" " \"silica_data\" folder with a description, screenshots, and an icon.") # Extract Control file and scripts from DEB - DpkgPy.control_extract(self, deb_path, self.root + "Packages/" + folder + - "/silica_data/scripts/") + suc = DpkgPy.control_extract(self, deb_path, self.root + "Packages/" + folder + + "/silica_data/scripts/") + print(suc) # Remove the Control; it's not needed. - os.remove(self.root + "Packages/" + folder + "/silica_data/scripts/Control") + os.remove(self.root + "Packages/" + folder + + "/silica_data/scripts/Control") if not os.listdir(self.root + "Packages/" + folder + "/silica_data/scripts/"): - os.rmdir(self.root + "Packages/" + folder + "/silica_data/scripts/") + os.rmdir(self.root + "Packages/" + + folder + "/silica_data/scripts/") else: print("Estimating dependencies...") # Use the filesystem to see if Zeppelin, Anemone, LockGlyph, XenHTML, and similar. @@ -486,29 +537,43 @@ def CheckForSilicaData(self): repo_settings = PackageLister.GetRepoSettings(self) # Ask for name - output['name'] = input("What should we name this package? ") + output['name'] = input( + "What should we name this package? ") # Automatically generate a bundle ID from the package name. - domain_breakup = repo_settings['cname'].split(".")[::-1] + domain_breakup = repo_settings['cname'].split(".")[ + ::-1] only_alpha_regex = re.compile('[^a-zA-Z]') - machine_safe_name = only_alpha_regex.sub("", output['name']).lower() - output['bundle_id'] = ".".join(str(x) for x in domain_breakup) + "." + machine_safe_name - output['tagline'] = input("What is a brief description of the package? ") - output['homepage'] = "https://" + repo_settings['cname'] + machine_safe_name = only_alpha_regex.sub( + "", output['name']).lower() + output['bundle_id'] = ".".join( + str(x) for x in domain_breakup) + "." + machine_safe_name + output['tagline'] = input( + "What is a brief description of the package? ") + output['homepage'] = "https://" + \ + repo_settings['cname'] # I could potentially default this to what is in settings.json but attribution may be an issue. - output['developer']['name'] = input("Who made this package? This is likely your name. ") - output['developer']['email'] = input("What is the author's email address? ") - output['works_min'] = input("What is the lowest iOS version the package works on? ") - output['works_max'] = input("What is the highest iOS version the package works on? ") - output['featured'] = input("Should this package be featured on your repo? (true/false) ") - PackageLister.CreateFolder(self, "Packages/" + folder + "/silica_data/") - PackageLister.CreateFile(self, "Packages/" + folder + "/silica_data/index.json", json.dumps(output)) + output['developer']['name'] = input( + "Who made this package? This is likely your name. ") + output['developer']['email'] = input( + "What is the author's email address? ") + output['works_min'] = input( + "What is the lowest iOS version the package works on? ") + output['works_max'] = input( + "What is the highest iOS version the package works on? ") + output['featured'] = input( + "Should this package be featured on your repo? (true/false) ") + PackageLister.CreateFolder( + self, "Packages/" + folder + "/silica_data/") + PackageLister.CreateFile( + self, "Packages/" + folder + "/silica_data/index.json", json.dumps(output)) def CompilePackages(self): """ Creates a Packages.bz2 file. """ # TODO: Update DpkgPy to generate DEB files without dependencies (for improved win32 support) - call(["dpkg-scanpackages", "-m", "."], cwd=self.root + "docs/", stdout=open(self.root + "docs/Packages", "w")) + call(["dpkg-scanpackages", "-m", "."], cwd=self.root + + "docs/", stdout=open(self.root + "docs/Packages", "w")) # For this, we're going to have to run it and then get the output. From here, we can make a new file. shutil.copy(self.root + "docs/Packages", self.root + "docs/Packages2") call(["bzip2", "Packages"], cwd=self.root + "docs/") @@ -534,8 +599,10 @@ def SignRelease(self): try: if repo_settings['enable_gpg'].lower() == "true": print("Signing repository with GPG...") - key = "Silica MobileAPT Repository" # Most of the time, this is acceptable. - call(["gpg", "-abs", "-u", key, "-o", "Release.gpg", "Release"], cwd=self.root + "docs/") + # Most of the time, this is acceptable. + key = "Silica MobileAPT Repository" + call(["gpg", "-abs", "-u", key, "-o", "Release.gpg", + "Release"], cwd=self.root + "docs/") print("Generated Release.gpg!") except Exception: pass diff --git a/util/DpkgPy.py b/util/DpkgPy.py index 512032b..4fc1162 100644 --- a/util/DpkgPy.py +++ b/util/DpkgPy.py @@ -1,6 +1,9 @@ -import arpy +import io +import lzma import tarfile - +import arpy +import os +import pyzstd class DpkgPy: """ @@ -22,30 +25,30 @@ def extract(self, input_path, output_path): try: root_ar = arpy.Archive(input_path) root_ar.read_all_headers() - try: - data_bin = root_ar.archived_files[b'data.tar.gz'] + + data_bin_ext = None + + for ext in ['.gz', '.lzma', '.xz', '.zst']: + if b'data.tar' + ext.encode() in root_ar.archived_files: + data_bin_ext = ext + break + + if data_bin_ext is None: + raise ValueError("Unsupported data_bin format") + + data_bin = root_ar.archived_files[b'data.tar' + data_bin_ext.encode()] + + if data_bin_ext == '.gz': data_tar = tarfile.open(fileobj=data_bin) - data_tar.extractall(output_path) - except Exception: - try: - data_theos_bin = root_ar.archived_files[b'data.tar.lzma'] - data_theos_bin.seekable = lambda: True - data_theos_tar = tarfile.open(fileobj=data_theos_bin, mode='r:xz') - data_theos_tar.extractall(output_path) - except Exception: - try: - data_theos_bin = root_ar.archived_files[b'data.tar.xz'] - data_theos_bin.seekable = lambda: True - data_theos_tar = tarfile.open(fileobj=data_theos_bin, mode='r:xz') - data_theos_tar.extractall(output_path) - except Exception: - print("\033[91m- DEB Extraction Error -\n" - "The DEB file inserted for one of your packages is invalid. Please report this as a bug " - "and attach the DEB file at \"" + output_path + "\".\033[0m") + elif data_bin_ext == '.lzma' or data_bin_ext == '.xz': + data_data = lzma.decompress(data_bin.read()) + data_tar = tarfile.open(fileobj=io.BytesIO(data_data)) + elif data_bin_ext == '.zst': + data_data = pyzstd.decompress(data_bin.read()) + data_tar = tarfile.open(fileobj=io.BytesIO(data_data)) - control_bin = root_ar.archived_files[b'control.tar.gz'] - control_tar = tarfile.open(fileobj=control_bin) - control_tar.extractall(output_path) + data_tar.extractall(output_path) + print(output_path) return True except Exception: return False @@ -60,8 +63,28 @@ def control_extract(self, input_path, output_path): try: root_ar = arpy.Archive(input_path) root_ar.read_all_headers() - control_bin = root_ar.archived_files[b'control.tar.gz'] - control_tar = tarfile.open(fileobj=control_bin) + + control_bin_ext = None + + for ext in ['.gz', '.lzma', '.xz', '.zst']: + if b'control.tar' + ext.encode() in root_ar.archived_files: + control_bin_ext = ext + break + + if control_bin_ext is None: + raise ValueError("Unsupported control_bin format") + + control_bin = root_ar.archived_files[b'control.tar' + control_bin_ext.encode()] + + if control_bin_ext == '.gz': + control_tar = tarfile.open(fileobj=control_bin) + elif control_bin_ext == '.lzma' or control_bin_ext == '.xz': + control_data = lzma.decompress(control_bin.read()) + control_tar = tarfile.open(fileobj=io.BytesIO(control_data)) + elif control_bin_ext == '.zst': + control_data = pyzstd.decompress(control_bin.read()) + control_tar = tarfile.open(fileobj=io.BytesIO(control_data)) + control_tar.extractall(output_path) return True except Exception: