test: add tests to bucket endpoint, mongo client and boto (#6)

* hotfix: update gitignore

* hotfix: update docker-compose file - remove env-files config

* hotfix: structured and enhancement project (#3)

* hotfix: update readme project

* feat: add new environment variable

* feat: add package beanie-odm

* fix: remove unsing config in docker-compose file

* feat: update botoclient function

* feat: cleanup error codes and add news

* feat: remove unusing functions and update utils functionnal

* feat: define bucket policy config

* test: add tests for endpoints and functional (#5)

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+known_third_party = beanie,boto3,botocore,fastapi,fastapi_pagination,httpx,mongomock_motor,motor,pydantic,pydantic_settings,pymongo,pytest,slugify,starlette,typer,uvicorn

src/common/mongo_client.py

@@ -1,8 +1,12 @@
+import logging
 from typing import Optional
 
 from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo.server_api import ServerApi
 
 _mongoclient: Optional[AsyncIOMotorClient] = None
+logging.basicConfig(format="%(message)s", level=logging.INFO)
+_log = logging.getLogger(__name__)
 
 
 async def config_mongodb_client(mongodb_uri: str) -> AsyncIOMotorClient:
@@ -15,5 +19,11 @@ async def config_mongodb_client(mongodb_uri: str) -> AsyncIOMotorClient:
     global _mongoclient
 
     if _mongoclient is None or _mongoclient.closed is True:
-        _mongoclient = AsyncIOMotorClient(mongodb_uri)
+        _mongoclient = AsyncIOMotorClient(mongodb_uri, server_api=ServerApi("1"))
+        try:
+            await _mongoclient.admin.command("ping")
+            _log.info("Pinged your deployment. You successfully connected to MongoDB")
+        except Exception as e:
+            _log.debug(f"Failed to connect to MongoDB: {e}")
+            _mongoclient = _mongoclient
     return _mongoclient

src/schemas/bucket.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import date
 from typing import Optional
 from pydantic import BaseModel, Field
 
@@ -11,5 +11,4 @@ class BucketSchema(BaseModel):
 class BucketFilter(BaseModel):
     bucket_name: Optional[str] = Field(None, title="Bucket name")
     description: Optional[str] = Field(None, title="Bucket description")
-    created_at: Optional[datetime] = Field(None, title="Bucket creation date")
-    updated_at: Optional[datetime] = Field(None, title="Bucket update date")
+    created_at: Optional[date] = Field(None, title="Bucket creation date")

src/services/bucket.py

@@ -22,7 +22,6 @@ async def create_new_bucket(bucket: BucketSchema, botoclient: boto3.client = Dep
     """
 
     bucket_name = format_bucket(bucket.bucket_name)
-
     try:
         botoclient.head_bucket(Bucket=bucket_name)
         raise CustomHTTPException(
@@ -52,7 +51,7 @@ async def create_new_bucket(bucket: BucketSchema, botoclient: boto3.client = Dep
         else:
             raise CustomHTTPException(
                 error_code=SfsErrorCodes.SFS_INVALID_NAME,
-                error_message=f"Error checking bucket: {str(exc)}",
+                error_message=str(exc),
                 status_code=status.HTTP_400_BAD_REQUEST,
             ) from exc
 
@@ -75,21 +74,16 @@ async def get_or_create_bucket(
     :rtype: Bucket
     """
 
-    if not await Bucket.find_one({"bucket_slug": bucket_name}).exists():
-        if create_bucket_if_not_exist:
-            return await create_new_bucket(bucket=BucketSchema(bucket_name=bucket_name), botoclient=botoclient)
-        else:
-            raise CustomHTTPException(
-                error_code=SfsErrorCodes.SFS_BUCKET_NOT_FOUND,
-                error_message=f"Bucket '{bucket_name}' not found.",
-                status_code=status.HTTP_404_NOT_FOUND,
-            )
-
-    if create_bucket_if_not_exist:
+    bucket = await Bucket.find_one({"bucket_slug": bucket_name})
+    if bucket is None and create_bucket_if_not_exist:
+        return await create_new_bucket(bucket=BucketSchema(bucket_name=bucket_name), botoclient=botoclient)
+    elif bucket and not create_bucket_if_not_exist:
+        return bucket
+    else:
         raise CustomHTTPException(
-            error_code=SfsErrorCodes.SFS_BUCKET_NAME_ALREADY_EXIST,
-            error_message=f"Bucket '{bucket_name}' already exists.",
-            status_code=status.HTTP_409_CONFLICT,
+            error_code=SfsErrorCodes.SFS_BUCKET_NOT_FOUND,
+            error_message=f"Bucket '{bucket_name}' not found.",
+            status_code=status.HTTP_400_BAD_REQUEST,
         )
 
 

tests/.test.env

@@ -6,8 +6,16 @@ APP_LOG_LEVEL=debug
 APP_HOSTNAME=0.0.0.0
 APP_ACCESS_LOG=True
 APP_DEFAULT_PORT=9090
-APP_TITLE="UNSTA: Simple file storage"
-APP_BUCKET_NAME_PATTERN=r'^[a-z0-9]{3,63}$'
+APP_TITLE="TEST: SFS"
+HASH_SECRET_KEY=bNFC4nO7
+
+# MODELS NAMES
+BUCKET_DB_COLLECTION=tests.buckets
+MEDIA_DB_COLLECTION=tests.media
+
+# MONGODB URI CONFIG
+MONGO_DB=tests
+MONGODB_URI=mongodb://root:secret@localhost:27017/?authMechanism=SCRAM-SHA-256
 
 # STORAGE SETTINGS
 STORAGE_HOST=http://local:9000

tests/common/test_boto_client.py

@@ -6,12 +6,14 @@ async def test_get_boto_client_success(mock_boto_client):
     from src.common.boto_client import get_boto_client
 
     mock_boto_instance = mock_boto_client.return_value
-    boto_instance = get_boto_client()
+    boto_client = get_boto_client()
 
-    assert mock_boto_client.call_args.kwargs == {
-        "endpoint_url": settings.STORAGE_HOST,
-        "aws_access_key_id": settings.STORAGE_ACCESS_KEY,
-        "aws_secret_access_key": settings.STORAGE_SECRET_KEY,
-        "region_name": settings.STORAGE_REGION_NAME,
-    }
-    assert boto_instance is mock_boto_instance
+    mock_boto_client.assert_called_once()
+    mock_boto_client.assert_called_once_with(
+        "s3",
+        endpoint_url=settings.STORAGE_HOST,
+        aws_access_key_id=settings.STORAGE_ACCESS_KEY,
+        aws_secret_access_key=settings.STORAGE_SECRET_KEY,
+        region_name=settings.STORAGE_REGION_NAME,
+    )
+    assert boto_client is mock_boto_instance

tests/conftest.py

@@ -1,7 +1,11 @@
 from unittest import mock
 
 import pytest
+from beanie import init_beanie
 from httpx import AsyncClient
+from mongomock_motor import AsyncMongoMockClient
+
+from src.config import settings
 
 
 @pytest.fixture
@@ -12,10 +16,22 @@ def fake_data():
 
 
 @pytest.fixture()
-async def mock_app():
-    from src.main import app
+async def mock_app_instance():
+    from src.main import app as mock_app
+
+    yield mock_app
+
+
+@pytest.fixture()
+def fixture_models():
+    from src import models
+
+    return models
+
 
-    yield app
+@pytest.fixture(autouse=True)
+async def mongo_client():
+    yield AsyncMongoMockClient()
 
 
 @pytest.fixture(autouse=True)
@@ -24,25 +40,35 @@ def mock_boto_client():
         yield mock_client
 
 
+@pytest.fixture(autouse=True)
+async def fixture_client_mongo(mock_app_instance, mongo_client, fixture_models):
+    mock_app_instance.mongo_db_client = mongo_client[settings.MONGO_DB]
+    await init_beanie(
+        database=mock_app_instance.mongo_db_client,
+        document_models=[fixture_models.Bucket, fixture_models.Media],
+    )
+    yield mongo_client
+
+
+@pytest.fixture()
+def bucket_data(fake_data):
+    return {"bucket_name": "unsta-storage", "description": fake_data.text()}
+
+
+@pytest.mark.asyncio
 @pytest.fixture()
-def mock_bucket_data(fake_data):
-    return {
-        "Buckets": [
-            {
-                "Name": fake_data.name(),
-                "CreationDate": fake_data.date_time_this_year(),
-            }
-        ]
-    }
+async def default_bucket(fixture_models, bucket_data):
+    result = await fixture_models.Bucket(**bucket_data).create()
+    return result
 
 
 @pytest.fixture
-async def http_client_api(mock_app, mock_boto_client):
+async def http_bucket_api(mock_app_instance, fixture_client_mongo, mock_boto_client):
     from src.common.boto_client import get_boto_client
 
-    mock_app.dependency_overrides[get_boto_client] = lambda: mock_boto_client
+    mock_app_instance.dependency_overrides[get_boto_client] = lambda: mock_boto_client
 
-    async with AsyncClient(app=mock_app, base_url="http://sfs.api") as ac:
-        yield ac
+    async with AsyncClient(app=mock_app_instance, base_url="http://sfs.api") as bucket_api:
+        yield bucket_api
 
-    mock_app.dependency_overrides = {}
+    mock_app_instance.dependency_overrides = {}

tests/database/test_mongo_client.py

@@ -0,0 +1,28 @@
+from unittest import mock
+
+import pytest
+
+from src.config import settings
+from src.config.database import shutdown_db_client, startup_db_client
+
+
+@pytest.mark.asyncio
+@mock.patch("src.config.database.init_beanie", return_value=None)
+async def test_startup_db_client(mock_init_beanie, fixture_client_mongo, mock_app_instance, fixture_models):
+    await startup_db_client(app=mock_app_instance, models=[fixture_models.Bucket, fixture_models.Media])
+
+    assert mock_app_instance.mongo_db_client is not None
+    assert fixture_client_mongo.is_mongos is True
+
+    mock_init_beanie.assert_called_once_with(
+        database=mock_app_instance.mongo_db_client[settings.MONGO_DB],
+        document_models=[fixture_models.Bucket, fixture_models.Media],
+        multiprocessing_mode=True,
+    )
+
+
+@pytest.mark.asyncio
+async def test_shutdown_db_client(mock_app_instance):
+    mock_app_instance.mongo_db_client = mock.AsyncMock()
+    await shutdown_db_client(app=mock_app_instance)
+    mock_app_instance.mongo_db_client.close.assert_called_once()