Skip to content

Commit

Permalink
chore(merge): merge commit for adding db entry during usersync
Browse files Browse the repository at this point in the history
  • Loading branch information
@Avantol13
Avantol13 committed Sep 12, 2018
1 parent 21e113c commit dd012f5
Showing 1 changed file with 151 additions and 93 deletions.
244 changes: 151 additions & 93 deletions fence/resources/storage/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,16 @@
from storageclient import get_client

from fence.models import (
CloudProvider, Bucket, ProjectToBucket, GoogleBucketAccessGroup, User
CloudProvider,
Bucket,
ProjectToBucket,
GoogleBucketAccessGroup,
User,
GoogleProxyGroupToGoogleBucketAccessGroup,
)
from fence.errors import NotSupported, InternalError, Unauthorized, NotFound
from fence.resources.google import STORAGE_ACCESS_PROVIDER_NAME
from fence.resources.google import STORAGE_ACCESS_PROVIDER_NAME as GOOGLE_PROVIDER


def check_exist(f):
@wraps(f)
Expand All @@ -21,56 +27,56 @@ def wrapper(self, provider, *args, **kwargs):

# NOTE: new storage privileges are expected to have -storage as a suffix
# ex: delete-storage
PRIVILEGES = [
"read-storage",
"write-storage",
"admin"
]
PRIVILEGES = ["read-storage", "write-storage", "admin"]


def get_endpoints_descriptions(providers, session):
desc = {}
for provider in providers:
if provider == 'cdis':
desc['/cdis'] = 'access to Gen3 APIs'
if provider == "cdis":
desc["/cdis"] = "access to Gen3 APIs"
else:
p = session.query(CloudProvider).filter_by(name=provider).first()
if p is None:
raise InternalError(
"{} is not supported by the system!".format(provider))
desc['/' + provider] = p.description or ''
"{} is not supported by the system!".format(provider)
)
desc["/" + provider] = p.description or ""
return desc


class StorageManager(object):

def __init__(self, credentials, logger):
self.logger = logger
self.clients = {}
for provider, config in credentials.iteritems():
if 'backend' not in config:
if "backend" not in config:
self.logger.error(
"Storage provider {} is not configured with backend"
.format(provider))
"Storage provider {} is not configured with backend".format(
provider
)
)
raise InternalError("Something went wrong")

backend = config['backend']
backend = config["backend"]
creds = copy.deepcopy(config)
del creds['backend']
del creds["backend"]
self.clients[provider] = get_client(config=config, backend=backend)

def check_auth(self, provider, user):
"""
check if the user should be authorized to storage resources
"""
storage_access = any([
'read-storage' in item
for item in user.project_access.values()
])
backend_access = any([
sa.provider.name == provider for p in user.projects.values()
for sa in p.storage_access
])
storage_access = any(
["read-storage" in item for item in user.project_access.values()]
)
backend_access = any(
[
sa.provider.name == provider
for p in user.projects.values()
for sa in p.storage_access
]
)
if storage_access and backend_access:
return True
else:
Expand Down Expand Up @@ -122,16 +128,21 @@ def create_bucket(self, provider, session, bucketname, project):
:param session: sqlalchemy session
:param bucketname: name of the bucket
"""
provider = session.query(CloudProvider).filter(
CloudProvider.name == provider).one()
bucket = session.query(Bucket).filter(
Bucket.name == bucketname).first()
provider = (
session.query(CloudProvider).filter(CloudProvider.name == provider).one()
)
bucket = session.query(Bucket).filter(Bucket.name == bucketname).first()
if not bucket:
bucket = Bucket(name=bucketname, provider=provider)
bucket = session.merge(bucket)
if not session.query(ProjectToBucket).filter(
if (
not session.query(ProjectToBucket)
.filter(
ProjectToBucket.bucket_id == bucket.id,
ProjectToBucket.project_id == project.id).first():
ProjectToBucket.project_id == project.id,
)
.first()
):
project_to_bucket = ProjectToBucket(bucket=bucket, project=project)
session.add(project_to_bucket)
c = self.clients[provider.name]
Expand All @@ -148,15 +159,15 @@ def grant_access(self, provider, username, project, access, session):
:param provider: storage backend provider
"""
access = self._get_valid_access_privileges(access)
storage_user = self._get_or_create_storage_user(
username, provider, session)
storage_user = self._get_or_create_storage_user(username, provider, session)

storage_username = StorageManager._get_storage_username(
storage_user, provider)
storage_username = StorageManager._get_storage_username(storage_user, provider)

if storage_username:
for b in project.buckets:
self._update_access_to_bucket(b, provider, storage_username, access)
self._update_access_to_bucket(
b, provider, storage_user, storage_username, access, session
)

@check_exist
def revoke_access(self, provider, username, project, session):
Expand All @@ -171,12 +182,13 @@ def revoke_access(self, provider, username, project, session):
if storage_user is None:
return

storage_username = StorageManager._get_storage_username(
storage_user, provider)
storage_username = StorageManager._get_storage_username(storage_user, provider)

if storage_username:
for b in project.buckets:
self._revoke_access_to_bucket(b, provider, storage_username)
self._revoke_access_to_bucket(
b, provider, storage_user, storage_username, session
)

@check_exist
def has_bucket_access(self, provider, user, bucket, access):
Expand All @@ -188,8 +200,9 @@ def has_bucket_access(self, provider, user, bucket, access):
access = self._get_valid_access_privileges(access)
storage_username = StorageManager._get_storage_username(user, provider)

return (storage_username and self.clients[provider].has_bucket_access(
bucket.name, storage_username))
return storage_username and self.clients[provider].has_bucket_access(
bucket.name, storage_username
)

@check_exist
def get_or_create_user(self, provider, user):
Expand Down Expand Up @@ -297,11 +310,10 @@ def _get_storage_user(self, username, provider, session):
Returns:
fence.models.User: User with username
"""
if provider == STORAGE_ACCESS_PROVIDER_NAME:
user = session.query(User).filter_by(username=username).first()
else:
user = self.clients[provider].get_user(username)
return user
if provider == GOOGLE_PROVIDER:
return session.query(User).filter_by(username=username).first()

return self.clients[provider].get_user(username)

def _get_or_create_storage_user(self, username, provider, session):
"""
Expand All @@ -319,71 +331,120 @@ def _get_or_create_storage_user(self, username, provider, session):
Returns:
fence.models.User: User with username
"""
if provider == STORAGE_ACCESS_PROVIDER_NAME:
if provider == GOOGLE_PROVIDER:
user = session.query(User).filter_by(username=username).first()
if not user:
raise NotFound(
'User not found with username {}. For Google Storage '
'Backend user\'s must already exist in the db and have a '
'Google Proxy Group.'
.format(username))
else:
user = self.clients[provider].get_or_create_user(username)
return user
"User not found with username {}. For Google Storage "
"Backend user's must already exist in the db and have a "
"Google Proxy Group.".format(username)
)
return user

return self.clients[provider].get_or_create_user(username)

def _update_access_to_bucket(
self, bucket, provider, storage_username, access):
self, bucket, provider, storage_user, storage_username, access, session
):
# Need different logic for google (since buckets can have multiple
# access groups)
if provider == STORAGE_ACCESS_PROVIDER_NAME:
if not bucket.google_bucket_access_groups:
raise NotFound(
'Google bucket {} does not have any access groups.'
.format(bucket.name))
if not provider == GOOGLE_PROVIDER:
self.clients[provider].add_bucket_acl(
bucket.name, storage_username, access=access
)
return

access = StorageManager._get_bucket_access_privileges(access)
if not bucket.google_bucket_access_groups:
raise NotFound(
"Google bucket {} does not have any access groups.".format(bucket.name)
)

for bucket_access_group in bucket.google_bucket_access_groups:
bucket_privileges = bucket_access_group.privileges or []
if set(bucket_privileges).issubset(access):
bucket_name = bucket_access_group.email

# NOTE: bucket_name for Google is the Google Access Group's
# email address.
# TODO Update storageclient API for more clarity
self.clients[provider].add_bucket_acl(
bucket_name, storage_username)
else:
# In the case of google, since we have multiple groups
# with access to the bucket, we need to also remove access
# here in case a users permissions change from read & write
# to just read.
bucket_name = bucket_access_group.email
self.clients[provider].delete_bucket_acl(
bucket_name, storage_username)
else:
self.clients[provider].add_bucket_acl(
bucket.name, storage_username, access=access)
access = StorageManager._get_bucket_access_privileges(access)

for bucket_access_group in bucket.google_bucket_access_groups:
bucket_privileges = bucket_access_group.privileges or []
if set(bucket_privileges).issubset(access):
bucket_name = bucket_access_group.email

# NOTE: bucket_name for Google is the Google Access Group's
# email address.
# TODO Update storageclient API for more clarity
self.clients[provider].add_bucket_acl(bucket_name, storage_username)

StorageManager._add_google_db_entry_for_bucket_access(
storage_user, bucket_access_group, session
)
else:
# In the case of google, since we have multiple groups
# with access to the bucket, we need to also remove access
# here in case a users permissions change from read & write
# to just read.
StorageManager._remove_google_db_entry_for_bucket_access(
storage_user, bucket_access_group, session
)

bucket_name = bucket_access_group.email
self.clients[provider].delete_bucket_acl(bucket_name, storage_username)

def _revoke_access_to_bucket(
self, bucket, provider, storage_username):
self, bucket, provider, storage_user, storage_username, session
):
# Need different logic for google (since buckets can have multiple
# access groups)
if provider == STORAGE_ACCESS_PROVIDER_NAME:
if provider == GOOGLE_PROVIDER:
for bucket_access_group in bucket.google_bucket_access_groups:
StorageManager._remove_google_db_entry_for_bucket_access(
storage_user, bucket_access_group, session
)
bucket_name = bucket_access_group.email
self.clients[provider].delete_bucket_acl(
bucket_name, storage_username)
self.clients[provider].delete_bucket_acl(bucket_name, storage_username)
else:
self.clients[provider].delete_bucket_acl(
bucket.name, storage_username)
self.clients[provider].delete_bucket_acl(bucket.name, storage_username)

@staticmethod
def _add_google_db_entry_for_bucket_access(
storage_user, bucket_access_group, session
):
"""
Add a db entry specifying that a given user has storage access
to the provided Google bucket access group
"""
storage_user_access_db_entry = GoogleProxyGroupToGoogleBucketAccessGroup(
proxy_group_id=storage_user.google_proxy_group_id,
access_group_id=bucket_access_group.id,
)
session.add(storage_user_access_db_entry)
session.commit()

# FIXME: create a delete() on GoogleProxyGroupToGoogleBucketAccessGroup and use here.
# previous attempts to use similar delete() calls on other models resulting in errors
# with mismatched sessions during testing
@staticmethod
def _remove_google_db_entry_for_bucket_access(
storage_user, bucket_access_group, session
):
"""
Remove the db entry specifying that a given user has storage access
to the provided Google bucket access group
"""
storage_user_access_db_entry = (
session.query(GoogleProxyGroupToGoogleBucketAccessGroup)
.filter_by(
proxy_group_id=storage_user.google_proxy_group_id,
access_group_id=bucket_access_group.id,
)
.first()
)
if storage_user_access_db_entry:
session.remove(storage_user_access_db_entry)
session.commit()

@staticmethod
def _get_storage_username(user, provider):
# Need different information for google (since buckets and
# users are represented with Google Groups)
username = None
if provider == STORAGE_ACCESS_PROVIDER_NAME:
if provider == GOOGLE_PROVIDER:
if user.google_proxy_group:
username = user.google_proxy_group.email
else:
Expand All @@ -409,8 +470,5 @@ def _get_bucket_access_privileges(access_list):
List(str): Simplified list of bucket privileges
"""
access = StorageManager._get_valid_access_privileges(access_list)
bucket_access = [
access_level.split('-')[0]
for access_level in access
]
bucket_access = [access_level.split("-")[0] for access_level in access]
return bucket_access

0 comments on commit dd012f5

Please sign in to comment.