Add GALAXY_LDAP_MIRROR_ONLY_EXISTING_GROUPS config option (#1609) (#1665

)

* Add GALAXY_LDAP_MIRROR_ONLY_EXISTING_GROUPS setting

Issue: AAH-2112

---------

Co-authored-by: Bruno Rocha <rochacbruno@gmail.com>
(cherry picked from commit f4710e9)

Co-authored-by: David Newswanger <dnewswan@redhat.com>

CHANGES/2112.feature

@@ -0,0 +1 @@
+Add GALAXY_LDAP_MIRROR_ONLY_EXISTING_GROUPS which configures LDAP to only add users into groups that already exist in the system.

dev/common/setup_test_data.py

@@ -95,6 +95,7 @@
         object_id=ns.id,
     )
 
+print("Create a signing namespace and roles")
 signing_ns, _ = Namespace.objects.get_or_create(name="signing")
 # connect group to role for this namespace object
 GroupRole.objects.get_or_create(
@@ -103,3 +104,6 @@
     content_type=ContentType.objects.get(model="namespace"),
     object_id=signing_ns.id,
 )
+
+print("Add a group that exists in the testing LDAP container")
+ldap_group, _ = Group.objects.get_or_create(name="admin_staff")

dev/standalone-ldap/RUN_INTEGRATION.sh

@@ -23,7 +23,9 @@ pip install -r integration_requirements.txt
 pip show epdb || pip install epdb
 
 echo "Setting up test data"
-docker exec -i galaxy_ng_api_1 /entrypoint.sh manage shell < dev/common/setup_test_data.py
+# If using a custom DEV_IMAGE_SUFFIX the container name will be different.
+CONTAINER_ID=${GALAXY_API_CONTAINER_NAME:-galaxy_ng_api_1}
+docker exec -i $CONTAINER_ID /entrypoint.sh manage shell < dev/common/setup_test_data.py
 
 
 #export HUB_API_ROOT='http://localhost:5001/api/'

dev/standalone-ldap/galaxy_ng.env

@@ -31,15 +31,19 @@ PULP_AUTH_LDAP_USER_SEARCH_SCOPE="SUBTREE"
 PULP_AUTH_LDAP_USER_SEARCH_FILTER="(uid=%(user)s)"
 PULP_AUTH_LDAP_GROUP_SEARCH_BASE_DN="ou=people,dc=planetexpress,dc=com"
 PULP_AUTH_LDAP_GROUP_SEARCH_SCOPE="SUBTREE"
-PULP_AUTH_LDAP_GROUP_SEARCH_FILTER = "(objectClass=Group)"
+PULP_AUTH_LDAP_GROUP_SEARCH_FILTER="(objectClass=Group)"
 PULP_AUTH_LDAP_GROUP_TYPE_CLASS="django_auth_ldap.config:GroupOfNamesType"
 PULP_AUTH_LDAP_GROUP_TYPE_PARAMS={name_attr="cn"}
 
 # User attribute settings
-PULP_AUTH_LDAP_ALWAYS_UPDATE_USER =true
+PULP_AUTH_LDAP_ALWAYS_UPDATE_USER=true
 PULP_AUTH_LDAP_USER_ATTR_MAP={first_name="givenName", last_name="sn", email="mail"}
-PULP_AUTH_LDAP_MIRROR_GROUPS=true
-PULP_AUTH_LDAP_MIRROR_GROUPS_EXCEPT=['system:partner-engineers']
+
+## The following 3 settings are mutually exclusive
+# PULP_AUTH_LDAP_MIRROR_GROUPS=true
+# PULP_AUTH_LDAP_MIRROR_GROUPS_EXCEPT=['system:partner-engineers']
+PULP_GALAXY_LDAP_MIRROR_ONLY_EXISTING_GROUPS=true
+
 PULP_AUTH_LDAP_USER_FLAGS_BY_GROUP__is_staff="cn=ship_crew,ou=people,dc=planetexpress,dc=com"
 PULP_AUTH_LDAP_USER_FLAGS_BY_GROUP__is_superuser="cn=admin_staff,ou=people,dc=planetexpress,dc=com"
 

docs/integration/ldap.md

@@ -36,6 +36,8 @@ ldapsearch -H ldap://localhost:10389 -x -b "ou=people,dc=planetexpress,dc=com" -
 The following settings can be added to either `/etc/pulp/settings.py` or exported as environment
 variables prefixed with `PULP_`.
 
+!!! note
+    when using AAP Platform installer the variables are set under `automationhub_ldap` and `ldap_extra_settings` on the installer inventory file.
 
 Example using environment variables:
 
@@ -61,14 +63,33 @@ PULP_TOKEN_AUTH_DISABLED=true
 
 ### Required setting
 
-`django_auth_ldap` must be included as the first authentication backend, there is a preset called
+For `django_auth_ldap` to be included as the first authentication backend, there is a preset called
 `ldap`
 
 ```bash
 PULP_AUTHENTICATION_BACKEND_PRESET=ldap
 ```
 
-> You can set it to `custom` if you really want to override `PULP_AUTHENTICATION_BACKENDS` variable.
+#### customizing the order of authentication backends 
+
+Set `PULP_AUTHENTICATION_BACKEND_PRESET` to `custom` if you really want to override `PULP_AUTHENTICATION_BACKENDS` variable, this might be useful if you need to change the order of evaluated backends.
+
+<details>
+<summary>Example</summary>
+
+
+```python
+AUTHENTICATION_BACKEND_PRESET='custom'
+# arrange the order
+AUTHENTICATION_BACKENDS=[
+  "django.contrib.auth.backends.ModelBackend",
+  "pulpcore.backends.ObjectRolePermissionBackend",
+  "galaxy_ng.app.auth.ldap.GalaxyLDAPBackend",
+]
+```
+
+</details>
+
 
 ### Required Specific django_auth_ldap settings
 
@@ -132,7 +153,23 @@ PULP_AUTH_LDAP_REQUIRE_GROUP='hub_users'
 # Only users belonging to this group will be allowed to login
 ```
 
-Mapping groups from LDAP to user attributes on Django:
+### Mirroring only existing groups on Hub
+
+```bash
+PULP_GALAXY_LDAP_MIRROR_ONLY_EXISTING_GROUPS=true
+```
+
+When set to `true` only groups that already exist on Hub will be mirrored,
+this means that users will login but not all the user groups from LDAP
+will be mirrored, the authentication backend will map to the user only
+the groups that matches the same name of groups existing in Hub.
+
+!!! note
+  When this option is set the variable `AUTH_LDAP_MIRROR_GROUPS` will
+  be automatically set to `true` and `AUTH_LDAP_MIRROR_GROUPS_EXCEPT` will default
+  to `false` regardless of the value you set for those 2 variables.
+
+### Mapping groups from LDAP to user attributes on Django:
 
 Ex: Users belonging to `admin_staff` on LDAP is `superuser` on Django.
 

docs/usage_guide/collections.md

@@ -0,0 +1,174 @@
+# Collections
+
+This guide is intended to walk you through the basics of managing collections in Galaxy NG. For information on writing and using collections within the broader ansible ecosystem, please refer to the documentation for:
+- [Using collections](https://docs.ansible.com/ansible/latest/dev_guide/developing_collections.html#developing-collections)
+- [Developing collections](https://docs.ansible.com/ansible/latest/dev_guide/developing_collections.html#developing-collections)
+
+## Namespaces
+
+Collection namespaces typically contain information about the publisher of a specific collection. To view the namespaces in the system, navigate to the namespaces page under Collections > Namespaces in the navigation menu. From here you can view the namespaces that you have access to manage under "My namespaces", as well as all of the namespaces under "All".
+
+New namespaces can be created from here. To create a namespace, click the "create" button and pick a new namespace name. 
+
+To edit a namespace, select the namespace from "My namespaces" and  click the "Edit namespace" button from the kebab in the header. 
+
+![Namespace kebab](assets/namespace_kebab.png)
+
+From here you can add links to documentation, manage user access, set a description and write a readme for your namespace from the Edit Resources tab.
+
+![Edit Namespaces](assets/edit_namespace.png)
+
+### Permissions
+
+Users can be given access to manage a namespace by going to the "Namespace owners" tab and selecting one or more groups to assign to the namespace. Each group can be given a role with customized sets of permissions. To see more information about creating roles, visit the RBAC guide.
+
+## Uploading Collections
+
+Before uploading a collection, you must create a namespace that matches the collections namespace. For example if I have an artifact named `foo-bar-1.2.3.tar.gz`, I need to create the `foo` namespace before it can be uploaded.
+
+Once a collection is uploaded, it may not appear on the search page right away. See [Approval](#approval) for details.
+
+### Via the UI
+
+Find your namespace under "My namespaces" on the namespaces page. Click "Upload collection" and select the tarball you want to upload from your computer and click "upload".
+
+![Upload Collection](assets/upload_collection.png)
+
+This will kick off a collection import and you'll be redirected to the import log. If there are any errors with the collection, this page will let you know.
+
+![Import Log](assets/import_log.png)
+
+You can view previous imports for a namespace by selecting "Imports" from the kebab on the namespaces page.
+
+### Via the CLI
+
+Before you can do this you need to [configure the ansible-galaxy CLI to connect to Galaxy](#configuring-the-ansible-galaxy-cli).
+
+To upload a collection run the following `ansible-galaxy collection publish` command:
+
+```shell
+$ ansible-galaxy collection publish newswangerd-main_collection-1.0.3.tar.gz
+Publishing collection artifact '/Users/dnewswan/code/collections/collection_demo/collection_demo/newswangerd-main_collection-1.0.3.tar.gz' to dev http://localhost:5001/api/galaxy/
+Collection has been published to the Galaxy server dev http://localhost:5001/api/galaxy/
+Waiting until Galaxy import task http://localhost:5001/api/galaxy/v3/imports/collections/e2dda51b-d176-43da-a2b7-86c87013197a/ has completed
+Collection has been successfully published and imported to the Galaxy server dev http://localhost:5001/api/galaxy/
+```
+
+### Approval
+
+Once a collection is uploaded, it will either end up in the Published repository, where it will be available for download, or the staging repository where it needs to be approved first depending on what you have configured for [GALAXY_REQUIRE_CONTENT_APPROVAL](/galaxy_ng/config/options#galaxy).
+
+If approval is required, click the "Collections > Approval" link in the navigation menu. This will give you a list of newly uploaded collections to review. If signing is enabled, you will also be promted to sign the collection.
+
+!!! note
+    You will need the "Modify Ansible repo content" permission to access the Approval dashboard. Make sure you have an account with the appropriate role.
+
+![Approval dashboard](assets/approval.png)
+
+## Installing Collections from Galaxy NG
+
+Before you can do this you need to [configure the ansible-galaxy CLI to connect to Galaxy](#configuring-the-ansible-galaxy-cli).
+
+Once you have your CLI configured, simply run:
+
+```shell
+ansible-galaxy collection install namespace.name:1.0.3
+```
+
+You can copy the command to install a collection by finding your collection under Collections > Collection:
+
+![copy download command](assets/copy_download_cmd.png)
+
+## Syncing Collections
+
+Collections can be synced from galaxy.ansible.com or console.redhat.com (with an AAP subscription from Red Hat). For information on syncing certified collections, refer to the Red Hat product documentation.
+
+To sync from galaxy navigate to the Collections > Repository Management page and select the Remote page. Select "Edit" from the kebab on the "community" remote and upload a requirements.yaml file.
+
+![upload requirements file](assets/requirements_yaml_upload.png)
+
+A requirements file looks something like this. You can use any [python version range specifier](https://peps.python.org/pep-0440/#version-specifiers) to select a range of collection versions you would like to sync.
+
+```yaml
+collections:
+    # sync all versions of a collection
+  - name: amazon.aws
+
+    # sync one version of a collection
+  - name: community.general
+    version: 1.0.0
+
+    # sync a list of specific versions
+  - name: community.aws
+    version: "==1.1.0,==1.2.0"
+
+    # sync all versions later than a specific version
+  - name: community.asa
+    version: ">=1.0.0"
+
+    # include a specific older version of community.asa
+  - name: community.asa
+    version: "=0.4.0"
+```
+
+Once you have your requirements file uploaded you can click "sync" to kick the job off. To view the status of the sync while its running, locate it in the Tasks tab.
+
+## Deleting and Deprecating Collections
+
+Collections can deleted or deprecated by selecting a collection from "Collections > Collection". Deleting a collection will permanently remove it from the system. Before a collection can be deleted, any collections that depend on it must be deleted first.
+
+Deprecation will mark the collection as deprecated and hide it from the collection search page, but will still make it available for download. This is useful for communicating that a collection shouldn't be used anymore, without making it inaccessible for projects that may still rely on it.
+
+![deleting and deprecating](assets/deleting.png)
+
+## Configuring the ansible-galaxy CLI
+
+Check out the [ansible-galaxy CLI docs](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html#configuring-the-ansible-galaxy-client) for more in depth information on configurating the CLI.
+
+To configure the CLI to work with Galaxy NG you'll need to configure one server for each repository you wish to download from as well as credentials for your user account. The list of available repositories can be viewed from the "Collections > Repository management" page.
+
+You can either user your username and password for authentication or get a token from "Collections > API token management".
+
+A full configuration for all the repositories will look something like this:
+
+```cfg title="~/.ansible.cfg"
+[galaxy]
+server_list = publishd, certified, community, staging, rejected
+
+[galaxy_server.published]
+url=http://localhost:5001/api/galaxy/
+username=admin
+password=admin
+
+[galaxy_server.certified]
+url=http://localhost:5001/api/galaxy/content/certified/
+token=0c69a0dc89ef555b9a4870270f18f8772d116efa
+
+[galaxy_server.community]
+url=http://localhost:5001/api/galaxy/content/community/
+token=0c69a0dc89ef555b9a4870270f18f8772d116efa
+
+[galaxy_server.staging]
+url=http://localhost:5001/api/galaxy/content/staging/
+token=0c69a0dc89ef555b9a4870270f18f8772d116efa
+
+[galaxy_server.rejected]
+url=http://localhost:5001/api/galaxy/content/rejected/
+token=0c69a0dc89ef555b9a4870270f18f8772d116efa
+```
+
+Not all of these are required. If you don't have any collections synced from galaxy.ansible.com or console.redhat.com, you may only need to configure the `published` repository. `staging` and `rejected` won't be useful to most users.
+
+### Configure upload repositories
+
+To upload a collection you have to have one of two things configured in your ansible.cfg file:
+
+- The `published` repository at `/api/galaxy/` (note `/api/galaxy/content/published/` won't work). This will allow uploading to any namespace.
+- One or more `inbound-<NAMESPACE>` repositories. This will allow you to upload collections for a single namespace. The url for this can be copied from the CLI configuration tab on "Collections > Namespaces".
+
+    Example for the `testing` namespace:
+    ```
+    [galaxy_server.inbound-testing]
+    url=http://localhost:5001/api/galaxy/content/inbound-testing/
+    token=0c69a0dc89ef555b9a4870270f18f8772d116efa
+    ```

galaxy_ng/app/api/ui/views/settings.py

@@ -23,6 +23,8 @@ def get(self, request, *args, **kwargs):
             "GALAXY_MINIMUM_PASSWORD_LENGTH",
             "GALAXY_AUTH_LDAP_ENABLED",
             "GALAXY_CONTAINER_SIGNING_SERVICE",
+            "GALAXY_LDAP_MIRROR_ONLY_EXISTING_GROUPS",
+            "GALAXY_LDAP_DISABLE_REFERRALS",
         ]
         data = {key: settings.as_dict().get(key, None) for key in keyset}
         return Response(data)

galaxy_ng/app/auth/ldap.py

@@ -0,0 +1,42 @@
+import logging
+from django_auth_ldap.backend import LDAPBackend, LDAPSettings
+from galaxy_ng.app.models.auth import Group
+from django.conf import settings
+
+
+log = logging.getLogger(__name__)
+
+
+class GalaxyLDAPSettings(LDAPSettings):
+
+    _mirror_groups = None
+    _cached_groups = None
+
+    @property
+    def MIRROR_GROUPS(self):
+        log.debug("Cached LDAP groups: %s", str(self._cached_groups))
+        if settings.get("GALAXY_LDAP_MIRROR_ONLY_EXISTING_GROUPS"):
+            self._cached_groups = (
+                self._cached_groups
+                or set(Group.objects.all().values_list("name", flat=True))
+            )
+            if isinstance(self._mirror_groups, (set, frozenset)):
+                return self._mirror_groups.union(self._cached_groups)
+            else:
+                return self._cached_groups
+
+        return self._mirror_groups
+
+    @MIRROR_GROUPS.setter
+    def MIRROR_GROUPS(self, val):
+        self._mirror_groups = val
+
+
+class GalaxyLDAPBackend(LDAPBackend):
+    """
+    Add option to make mirror group only work with exiting groups in
+    the db.
+    """
+
+    def __init__(self):
+        self.settings = GalaxyLDAPSettings(self.settings_prefix, self.default_settings)

galaxy_ng/app/dynaconf_hooks.py

@@ -494,6 +494,10 @@ def configure_ldap(settings: Dynaconf) -> Dict[str, Any]:
             connection_options[ldap.OPT_REFERRALS] = 0
         data["AUTH_LDAP_CONNECTION_OPTIONS"] = connection_options
 
+        if settings.get("GALAXY_LDAP_MIRROR_ONLY_EXISTING_GROUPS"):
+            data["AUTH_LDAP_MIRROR_GROUPS"] = True
+            data["AUTH_LDAP_MIRROR_GROUPS_EXCEPT"] = None
+
     return data
 
 

galaxy_ng/app/settings.py

@@ -219,7 +219,7 @@
 AUTHENTICATION_BACKEND_PRESET = 'local'  # 'ldap' or 'keycloak' or 'local' or 'custom'
 AUTHENTICATION_BACKEND_PRESETS_DATA = {
     'ldap': [
-        "django_auth_ldap.backend.LDAPBackend",
+        "galaxy_ng.app.auth.ldap.GalaxyLDAPBackend",
         "django.contrib.auth.backends.ModelBackend",
         "pulpcore.backends.ObjectRolePermissionBackend"
     ],
@@ -237,3 +237,10 @@
 SOCIAL_AUTH_GITHUB_API_URL = os.environ.get('SOCIAL_AUTH_GITHUB_BASE_URL', 'https://api.github.com')
 SOCIAL_AUTH_GITHUB_KEY = os.environ.get('SOCIAL_AUTH_GITHUB_KEY')
 SOCIAL_AUTH_GITHUB_SECRET = os.environ.get('SOCIAL_AUTH_GITHUB_SECRET')
+
+
+# When set to True, galaxy will only load ldap groups into local
+# groups that already exist in the database. Ex: if user with ldap
+# groups foo and bar login and only group foo exists in the system,
+# the user will be added to foo and bar will be ignored.
+GALAXY_LDAP_MIRROR_ONLY_EXISTING_GROUPS = False