Fix treatment of unmanaged libraries (#1680)

* Optimize managed library detection

This commit creates a list of orphaned managed libraries by checking each
libraries installer via InstalledLibrary object model instead of performing
filesystem traversal twice (to subtract unmanaged ones).

* Fix unmanaged libraries being upgraded

This commit makes sure not to upgrade unmanaged libraries.

* Satisfy libraries in sorted order

... for better readability of debug logs.

* Fix adding INSTALLER file

This commit ensures to add <libname>.dist-info/INSTALLER and related RECORD
entry to mark installed WHEELS as managed.

package_control/distinfo.py

@@ -222,6 +222,28 @@ def generate_installer(self):
 
         return "Package Control\n"
 
+    def add_installer_to_record(self):
+        R"""
+        Add INSTALLER entry to .dist-info/RECORD file.
+
+        Note: hash has been pre-compiled using...
+
+        ```py
+        digest = hashlib.sha256("Package Control\n".encode("utf-8")).digest()
+        sha = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("utf-8")
+        ```
+        """
+        installer = self.dir_name + "/INSTALLER,"
+        record = self.abs_path("RECORD")
+
+        # make sure not to add duplicate entries
+        with open(record, "r", encoding="utf-8") as fobj:
+            items = [item for item in fobj.readlines() if not item.startswith(installer)]
+            items.append(installer + "sha256=Hg_Q6w_I4zpFfb6C24LQdd4oTAMHJZDk9gtuV2yOgkw,16\n")
+
+        with open(record, "w", encoding="utf-8", newline="\n") as fobj:
+            fobj.writelines(sorted(items))
+
     def generate_record(self, package_dirs, package_files):
         """
         Generates the .dist-info/RECORD file contents
@@ -378,9 +400,11 @@ def read_installer(self):
         :returns:
             An unicode string of of which installer was used.
         """
-
-        with open(self.abs_path("INSTALLER"), "r", encoding="utf-8") as fobj:
-            return fobj.readline().strip()
+        try:
+            with open(self.abs_path("INSTALLER"), "r", encoding="utf-8") as fobj:
+                return fobj.readline().strip()
+        except FileNotFoundError:
+            return ""
 
     def write_installer(self):
         """

package_control/library.py

@@ -141,6 +141,12 @@ def __init__(self, install_root, dist_info_dir, python_version):
         self.dist_name = dist_info_dir[: dist_info_dir.find("-")].lower()
         self.python_version = python_version
 
+    def is_managed(self):
+        """
+        Library was installed and is therefore managed by Package Control.
+        """
+        return self.dist_info.read_installer() == self.dist_info.generate_installer().strip()
+
 
 def list_all():
     """

package_control/package_cleanup.py

@@ -97,6 +97,7 @@ def run(self):
         removed_packages = None
 
         self.remove_legacy_libraries()
+        self.autofix_missing_installer()
 
         # Check metadata to verify packages were not improperly installed
         self.migrate_incompatible_packages(found_packages)
@@ -446,6 +447,47 @@ def install_missing_packages(self, found_packages):
             with ActivityIndicator('Installing missing packages...') as progress:
                 self.run_install_tasks(tasks, progress, unattended=True, package_kind='missing')
 
+    def autofix_missing_installer(self):
+        """
+        Add missing INSTALLER file to available unmodified libraries
+
+        Libraries without INSTALLER are treated unmanaged
+        and thus are not upgraded or removed automatically.
+
+        Package Control prior to v4.0.7 didn't add INSTALLER for
+        installed whl files. This option lets users fix it, manually.
+        """
+
+        available_libraries = self.manager.list_available_libraries()
+        for lib in self.manager.list_libraries():
+            installer = lib.dist_info.read_installer()
+            if installer:
+                continue
+
+            available_library = available_libraries.get(lib.dist_name)
+            if not available_library:
+                continue
+
+            if not any(
+                lib.python_version in available_release['python_versions']
+                for available_release in available_library['releases']
+            ):
+                continue
+
+            if not lib.dist_info.verify_files():
+                console_write(
+                    'Library "%s" for Python %s was modified, skipping!',
+                    (lib.name, lib.python_version)
+                )
+                continue
+
+            lib.dist_info.write_installer()
+            lib.dist_info.add_installer_to_record()
+            console_write(
+                'Library "%s" for Python %s fixed!',
+                (lib.name, lib.python_version)
+            )
+
     def remove_legacy_libraries(self):
         """
         Rename .dist-info directory

package_control/package_manager.py

@@ -958,8 +958,8 @@ def find_orphaned_libraries(self, required_libraries=None):
         installed_libraries = self.list_libraries()
         if required_libraries is None:
             required_libraries = self.find_required_libraries()
-        unmanaged_libraries = library.list_unmanaged()
-        return installed_libraries - required_libraries - unmanaged_libraries
+
+        return set(lib for lib in installed_libraries - required_libraries if lib.is_managed())
 
     def _download_zip_file(self, name, url, sha256=None):
         try:
@@ -1128,7 +1128,7 @@ def install_libraries(self, libraries, fail_early=True):
         """
 
         error = False
-        for lib in libraries:
+        for lib in sorted(libraries):
             if not self.install_library(lib):
                 if fail_early:
                     return False
@@ -1158,6 +1158,13 @@ def install_library(self, lib):
                 installed_version = pep440.PEP440Version(installed_version)
 
         is_upgrade = installed_library is not None
+        if is_upgrade and not installed_library.is_managed():
+            if debug:
+                console_write(
+                    'The library "%s" for Python %s was not installed by Package Control; leaving alone',
+                    (lib.name, lib.python_version)
+                )
+            return True
 
         release = None
         available_version = None
@@ -1268,6 +1275,9 @@ def install_library(self, lib):
                     )
                     return False
 
+                temp_did.write_installer()
+                temp_did.add_installer_to_record()
+
                 try:
                     temp_did.verify_python_version(lib.python_version)
                 except EnvironmentError as e:
@@ -1344,7 +1354,7 @@ def cleanup_libraries(self, required_libraries=None):
         orphaned_libraries = self.find_orphaned_libraries(required_libraries)
 
         error = False
-        for lib in orphaned_libraries:
+        for lib in sorted(orphaned_libraries):
             if not self.remove_library(lib):
                 error = True