Merge pull request #84 from pepkit/dev

v0.4.0

.github/workflows/pytest.yml

@@ -2,18 +2,29 @@ name: Run pytests
 
 on:
   push:
-    branches: [dev]
+    branches: [master, dev]
   pull_request:
     branches: [master, dev]
 
 jobs:
   pytest:
-    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-        os: [ubuntu-20.04]
-
+        python-version: ["3.8", "3.11"]
+        os: [ubuntu-latest] # can't use macOS when using service containers or container jobs
+        r: [release]
+    runs-on: ${{ matrix.os }}
+    services:
+        postgres:
+          image: postgres
+          env:
+            POSTGRES_USER: postgres
+            POSTGRES_PASSWORD: docker
+            POSTGRES_DB: pep-db
+            POSTGRES_HOST: localhost
+          ports:
+          - 5432:5432
+          options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
     - uses: actions/checkout@v2
 

Makefile

@@ -0,0 +1,14 @@
+lint:
+	# black should be last in the list, as it lint the code. Tests can fail if order will be different
+	flake8 && isort . && black .
+
+run-coverage:
+	coverage run -m pytest
+
+html-report:
+	coverage html
+
+open-coverage:
+	cd htmlcov && google-chrome index.html
+
+coverage: run-coverage html-report open-coverage

README.md

@@ -1,16 +1,15 @@
 # pepdbagent
 
-pepdbagent is a python library and toolkit that gives a user user-friendly 
+`pepdbagent` is a Python library and toolkit that gives a user-friendly 
 interface to connect, upload, update and retrieve information from pep-db.
 
-**pep-db** is and postgres database created for storing [PEPs](http://pep.databio.org/en/latest/). 
-It is a backend database for PEPhub. database enables storing huge projects and provides fast speed of retrieving them.
+**pep-db** is a postgres database created for storing [PEPs](http://pep.databio.org/en/latest/). 
+It is a backend database for PEPhub. 
 
-Before using pepdbagent, you should install or have access to pep-db.
+Before using pepdbagent, you should install or have access to a pep-db instance.
 
-To install pep-db you can use this tutorial:
+To run a pep-db instance, you can use this tutorial:
 - [pep-db installation](./docs/db_tutorial.md)
 
-
-pepdbagent tutorial is here:
+Then, follow the `pepdbagent` tutorial here:
 - [pedbagent](./docs/README.md)

docs/README.md

@@ -10,6 +10,7 @@ to increase readability, maintainability, and user experience of pepdbagent, whi
 PEPDatabaseAgent is the primary class that you will use. It connects to the database (using **BaseConnection** class).
 
 Example: Instiantiate a PEPDatabaseAgent object and connect to database:
+
 ```python
 
 import pepdbagent
@@ -30,9 +31,12 @@ Example:
 ```python
 import peppy
 
-prj_obj = peppy.Project("/path/to/project_config.yaml")
+prj_obj = peppy.Project("sample_pep/basic/project_config.yaml")
 
 # create a project
+namespace = "demo"
+name = "basic_project"
+tag = None
 agent.project.create(prj_obj, namespace, name, tag)
 
 # updating record in database (project)
@@ -101,7 +105,8 @@ AnnotationRetrunModel(count=1,
                                                description=None, 
                                                last_update_date='2022-11-09', 
                                                submission_date='2023-01-09', 
-                                               digest='36bb973f2eca3706ed9852abddd')
+                                               digest='36bb973f2eca3706ed9852abddd',
+                                               pep_schema="bedmake")])
 ```
 
 
@@ -128,4 +133,8 @@ NamespaceReturnModel(count=1,
                      limit=100, 
                      offset=0, 
                      results=[NamespaceResultModel(namespace='databio', number_of_projects=6, number_of_samples=470)])
-```
+```
+
+
+# Example PEPs
+To populate database with example peps use function written in manual tests: [Manual test](../manual_tests.py)

docs/changelog.md

@@ -2,9 +2,22 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
+## [0.4.0] -- 2023-06-09
+- Transitioned to SQLAlchemy ORM.
+- Added a pep_schema column to the database.
+- Implemented a new testing approach.
+- Integrated sorting functionality into the annotations module.
+- Temporarily disabled description-based searches to mitigate long processing times and reduce database load.
+- Included timezone support in the database.
+- Standardized namespace and names to lowercase for case-insensitivity.
+- Streamlined the database schema creation process, with tables now created dynamically as needed.
 
-## [0.3.0] -- 2023-01-19
 
+## [0.3.1] -- 2023-03-23
+- Fixed bug with peppy const dependencies
+
+
+## [0.3.0] -- 2023-01-19
 - Restructured pepdbagent: 
   - Renamed `Agent` class to `PEPDatabaseAgent`
   - created subclasses (project, annotation, namespace).

docs/db_tutorial.md

@@ -14,4 +14,8 @@ Now db is installed
 
 `docker exec -it 65f bash`
 
-`psql -U postgres -d pep-db`
+`psql -U postgres -d pep-db`
+
+---
+If you have your own database, you can initialize a connection using pepdbagent. 
+The pepdbagent will create a new database schema if it doesn't already exist, or throw an exception if the schema is incorrect.

manual_tests.py

@@ -1,25 +1,69 @@
 # file for manual local tests
 import peppy
-
+import os
 import pepdbagent
 from peppy import Project
 
 
-con = pepdbagent.PEPDatabaseAgent(dsn="postgresql://postgres:docker@localhost:5432/pep-db")
+DATA_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    "tests",
+    "data",
+)
+
+
+def get_path_to_example_file(namespace, project_name):
+    return os.path.join(DATA_PATH, namespace, project_name, "project_config.yaml")
+
+
+def list_of_available_peps():
+    pep_namespaces = os.listdir(DATA_PATH)
+    projects = {}
+    for np in pep_namespaces:
+        pep_name = os.listdir(os.path.join(DATA_PATH, np))
+        projects[np] = {p: get_path_to_example_file(np, p) for p in pep_name}
+    return projects
+
+
+def upload_sample_pep_to_db(connection: pepdbagent.PEPDatabaseAgent):
+    list_of_peps = list_of_available_peps()
+    for namespace, item in list_of_peps.items():
+        if namespace == "private_test":
+            private = True
+        else:
+            private = False
+        for name, path in item.items():
+            prj = peppy.Project(path)
+            connection.project.create(
+                namespace=namespace,
+                name=name,
+                tag="default",
+                is_private=private,
+                project=prj,
+                overwrite=True,
+                pep_schema="random_schema_name",
+            )
+
+    return None
+
+
+# populate database with few peps:
+con = pepdbagent.PEPDatabaseAgent(dsn="postgresql://postgres:docker@localhost:5432/pep-db", echo=False)
+upload_sample_pep_to_db(con)
+
+
 ###############
-# Upload
+# # Upload
+
 prj = peppy.Project(
-    "/home/bnt4me/virginia/repos/pepdbagent/sample_pep/basic/project_config.yaml"
+    "/home/bnt4me/virginia/repos/pepdbagent/tests/data/namespace1/basic/project_config.yaml"
 )
-con.project.create(project=prj, namespace="Khoroshevskyi", name="dupa", tag="test1", overwrite=True)
-
-# Project
+con.project.create(project=prj, namespace="dog_namespace", name="testttt", tag="test1", overwrite=True)
 
-prj_dow = con.project.get(namespace="Khoroshevskyi", name="dupa", tag="test1")
+con.project.exists(namespace="dog_namespace", name="testttt", tag="test1")
 
-print(prj_dow.name)
 
-prj_raw = con.project.get(namespace="Khoroshevskyi", name="dupa", tag="test1", raw=True)
+prj_raw = con.project.get(namespace="dog_namespace", name="testttt", tag="test1", raw=True)
 
 print(prj_raw)
 
@@ -29,27 +73,27 @@
 
 dd_list = con.annotation.get_by_rp(
     [
-        "Khoroshevskyi/gse_yaml:default",
-        "Khoroshevskyi/gse_yaml:default",
-        "Khoroshevskyi/dupa:f1",
+        "dog_namespace/gse_yaml:default",
+        "dog_namespace/gse_yaml:default",
+        "dog_namespace/testttt:f1",
     ],
-    admin="Khoroshevskyi",
+    admin="dog_namespace",
 )
 dd_list_private = con.annotation.get_by_rp(
     [
-        "Khoroshevskyi/gse_yaml:default",
-        "Khoroshevskyi/gse_yaml:default",
-        "Khoroshevskyi/dupa:f1",
+        "dog_namespace/gse_yaml:default",
+        "dog_namespace/gse_yaml:default",
+        "dog_namespace/testttt:f1",
     ]
 )
 
-dd_search = con.annotation.get(namespace="Khoroshevskyi")
-dd_search_pr = con.annotation.get(namespace="Khoroshevskyi", admin="Khoroshevskyi")
+dd_search = con.annotation.get(namespace="dog_namespace")
+dd_search_pr = con.annotation.get(namespace="dog_namespace", admin="dog_namespace")
 dd_search_pr_namespace = con.annotation.get(
-    query="s", admin=["Khoroshevskyi", "test_11"]
+    query="s", admin=["dog_namespace", "test_11"]
 )
 
-dd_all = con.annotation.get(admin=["Khoroshevskyi", "test_11"])
+dd_all = con.annotation.get(admin=["dog_namespace", "test_11"])
 
 
 print(dd_list)
@@ -64,15 +108,10 @@
 ################
 # Namespace
 
-ff = con.namespace.get("Khoroshevskyi", admin="Khoroshevskyi")
-
-print(ff)
-
-
-ff = con.project.get_by_rp("Khoroshevskyi/gse_yaml:default")
+ff = con.namespace.get("dog_namespace", admin="dog_namespace")
 
 print(ff)
 
-dell = con.project.delete(namespace="Khoroshevskyi", name="dupa", tag="test1")
+con.project.update(update_dict={"is_private": False}, namespace="dog_namespace", name="testttt", tag="test1")
 
-con.project.update(update_dict={"is_private": False}, namespace="Khoroshevskyi", name="dupa", tag="test1")
+dell = con.project.delete(namespace="dog_namespace", name="testttt", tag="test1")

pep_db/Dockerfile

@@ -2,4 +2,4 @@ FROM postgres
 ENV POSTGRES_USER postgres
 ENV POSTGRES_PASSWORD docker
 ENV POSTGRES_DB pep-db
-COPY pep_db.sql /docker-entrypoint-initdb.d/
+#COPY pep_db.sql /docker-entrypoint-initdb.d/

pep_db/pep_db.sql

@@ -16,6 +16,7 @@ CREATE TABLE projects (
     number_of_samples int NOT NULL,
     submission_date timestamp NOT NULL,
     last_update_date timestamp NOT NULL,
+    pep_schema TEXT,
     CONSTRAINT id PRIMARY KEY (namespace, name, tag)
 );
 

pepdbagent/__init__.py

@@ -1,10 +1,9 @@
 """ Package-level data """
-from .pepdbagent import *
-from ._version import __version__
-
-import logmuse
 import coloredlogs
+import logmuse
 
+from ._version import __version__
+from .pepdbagent import *
 
 _LOGGER = logmuse.init_logger("pepdbagent")
 coloredlogs.install(

pepdbagent/_version.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.4.0"