diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c1dc8e..f4cfb56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: diff --git a/.gitignore b/.gitignore index 68bc17f..981818a 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +.vscode/ +examples/basic_usage.ipynb diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 134d167..f111db8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: rev: 6.0.0 hooks: - id: flake8 - args: [--count, --show-source, --statistics, '--ignore', 'E501,E203,W503'] + args: [--count, --show-source, --statistics, '--ignore', 'E501,E203,W503,E201,E202,E221,E222,E231,E241,E271,E272,E702,E713'] # additional_dependencies: [flake8-bugbear==21.3.1, pep8-naming] log_file: flake8.log diff --git a/.pylintrc b/.pylintrc index 7bef9d0..070db46 100644 --- a/.pylintrc +++ b/.pylintrc @@ -76,6 +76,7 @@ disable= no-name-in-module, dangerous-default-value, too-many-public-methods, too-many-locals, + no-value-for-parameter, diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..007bac7 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,24 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +sphinx: + configuration: docs/conf.py + +formats: + - pdf + - epub + +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/Dockerfile.docs b/Dockerfile.docs new file mode 100644 index 0000000..39a87bf --- /dev/null +++ b/Dockerfile.docs @@ -0,0 +1,23 @@ +FROM python:3.10-buster + +RUN apt-get update && apt-get install -y \ + pandoc default-jre graphviz \ + texlive-latex-recommended \ + texlive-latex-extra \ + texlive-fonts-recommended \ + latexmk + +WORKDIR /app +COPY . . + +RUN python -m pip install --upgrade pip + +RUN python -m pip install -e .[docs] --no-cache + +CMD sphinx-autobuild --host 0.0.0.0 docs/ docs/_build/html + +# Build: +# $ docker build -f Dockerfile.docs -t dsms-sdk-docs . + +# Run: +# $ docker run -it --rm -v $PWD:/app -p 8000:8000 dsms-sdk-docs diff --git a/README.md b/README.md index 7703aba..d81c8da 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,6 @@ The SDK provides a general Python interface to a remote DSMS deployment, allowin For the basic usage, please have a look on the Jupyter Notebook under `examples/basic_usage.ipynb`. This tutorial provides a basic overview of using the dsms package to interact with Knowledge Items. - ## Disclaimer Copyright (c) 2014-2024, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. acting on behalf of its Fraunhofer IWM. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..caa42d9 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/AWSSDKforPHP.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/AWSSDKforPHP.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/AWSSDKforPHP" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/AWSSDKforPHP" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..a44ef4c --- /dev/null +++ b/docs/README.md @@ -0,0 +1,65 @@ +# DSMS-SDK Docs + +If you find any error or problem with the documentation, please create an issue [in this repository](https://github.com/MI-FraunhoferIWM/dsms-python-sdk/issues). + +## Local Rendering + + +### HTML +A server will start, generate the docs and listen for changes in the source files. +This can be done by using docker or installing the development environment directly on the you machine. Next are installation guides for Docker and Linux OS. + +#### Docker +First, build the Docker image by running the following command: +```shell +$ docker build -f Dockerfile.docs -t dsms-sdk-docs . +``` + +Then, start the program by running: +```shell +$ docker run -it --rm -v $PWD:/app -p 8000:8000 dsms-sdk-docs +``` + +#### Linux +At an OS level (these commands work on Linux Debian): +```shell +$ sudo apt install pandoc graphviz default-jre +$ sudo apt-get install texlive-latex-recommended \ + texlive-latex-extra \ + texlive-fonts-recommended \ + latexmk +``` +The python dependencies: +```shell +$ pip install -e .[docs] +``` + +Now you can start the server and render the docs: +``` +$ sphinx-autobuild docs/source docs/build/html +``` +The documentation will be available on [`http://127.0.0.1:8000`](http://127.0.0.1:8000). + +#### VSCode + +To render the documentation using VSCode, follow these steps: + +1. **Ensure Live Server Extension is Installed:** + + You need to have the Live Server extension installed in VSCode. If you don't have it installed, you can find it [here](https://marketplace.visualstudio.com/items?itemName=ritwickdey.LiveServer). + +2. **Build the Documentation:** + + Open your terminal in VSCode and run the following command to build the HTML documentation: + +```shell +$ make html +``` +3. **Open the Documentation with Live Server:** + + After building the documentation, navigate to the docs/build/html directory in VSCode. + Right-click on the index.html file and select the option "Open with Live Server". + +4. **Access the Documentation:** + + The documentation will open in your default web browser and be accessible at http://127.0.0.1:5500. diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000..4f2a27a --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,5 @@ +/* Custom CSS for wrapping text in table cells */ +table td { + word-wrap: break-word; + white-space: normal !important; +} diff --git a/docs/assets/images/DSMS.jpg b/docs/assets/images/DSMS.jpg new file mode 100644 index 0000000..d1935f8 Binary files /dev/null and b/docs/assets/images/DSMS.jpg differ diff --git a/docs/assets/images/DSMS_Mod.jpg b/docs/assets/images/DSMS_Mod.jpg new file mode 100644 index 0000000..a360c15 Binary files /dev/null and b/docs/assets/images/DSMS_Mod.jpg differ diff --git a/docs/assets/images/DSMS_SDK.jpg b/docs/assets/images/DSMS_SDK.jpg new file mode 100644 index 0000000..b7f47b2 Binary files /dev/null and b/docs/assets/images/DSMS_SDK.jpg differ diff --git a/docs/assets/images/DSMS_logo.png b/docs/assets/images/DSMS_logo.png new file mode 100644 index 0000000..0a23baf Binary files /dev/null and b/docs/assets/images/DSMS_logo.png differ diff --git a/docs/assets/images/UML_KItem_schema.jpg b/docs/assets/images/UML_KItem_schema.jpg new file mode 100644 index 0000000..2a10e6d Binary files /dev/null and b/docs/assets/images/UML_KItem_schema.jpg differ diff --git a/docs/assets/images/copy_token_1.jpg b/docs/assets/images/copy_token_1.jpg new file mode 100644 index 0000000..7f2b4aa Binary files /dev/null and b/docs/assets/images/copy_token_1.jpg differ diff --git a/docs/assets/images/copy_token_2.jpg b/docs/assets/images/copy_token_2.jpg new file mode 100644 index 0000000..9946b15 Binary files /dev/null and b/docs/assets/images/copy_token_2.jpg differ diff --git a/docs/assets/images/copy_token_3.jpg b/docs/assets/images/copy_token_3.jpg new file mode 100644 index 0000000..7c4d2b2 Binary files /dev/null and b/docs/assets/images/copy_token_3.jpg differ diff --git a/docs/assets/images/copy_token_4.jpg b/docs/assets/images/copy_token_4.jpg new file mode 100644 index 0000000..90e33e3 Binary files /dev/null and b/docs/assets/images/copy_token_4.jpg differ diff --git a/docs/assets/images/copy_token_5.jpg b/docs/assets/images/copy_token_5.jpg new file mode 100644 index 0000000..734a3e6 Binary files /dev/null and b/docs/assets/images/copy_token_5.jpg differ diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..ff9c5e3 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,102 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +# -- Project information ----------------------------------------------------- + +project = "DSMS Documentation" +copyright = "2024, Materials Informatics Team at Fraunhofer IWM" +author = "Materials Informatics, Fraunhofer IWM" +release = "v1.0.0" + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. + +extensions = [ + "myst_parser", # Markdown support + "sphinx.ext.autodoc", # Include documentation from docstrings + "sphinx.ext.napoleon", # Google-style docstrings + "sphinx.ext.viewcode", # Add links to highlighted source code + "sphinx.ext.graphviz", # Add support for graphviz + "sphinxcontrib.plantuml", # Add support for plantuml + "sphinx_copybutton", # Add copy button to code blocks + "nbsphinx", # Add support for Jupyter Notebooks + "IPython.sphinxext.ipython_console_highlighting", # Add syntax highlighting for IPython + "sphinx.ext.autosectionlabel", # Add support for autolabeling sections + "sphinx_panels", # Add support for panels + "sphinx_markdown_tables", # Add support for markdown tables + "sphinxcontrib.redoc", # Add support for redoc +] + +master_doc = "index" +myst_enable_extensions = ["colon_fence"] + +plantuml = "java -jar lib/plantuml.jar" +plantuml_output_format = "svg_img" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] +html_static_path = ["assets"] + + +def setup(app): + app.add_css_file("custom.css") + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. + +html_theme = "sphinx_book_theme" +html_logo = "assets/images/DSMS_logo.png" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# "style_nav_header_background": "#4472c4", : Blue of DSMS +# "style_nav_header_background": "#109193", : Green of DSMS +html_static_path = ["_static"] +templates_path = ["_templates"] +html_theme_options = { + "use_download_button": True, +} + +nbsphinx_allow_errors = True +nbsphinx_execute = "never" + +# -- Options for LaTeX output ------------------------------------------------- +latex_documents = [ + ( + "index", + "dsms_docs.tex", + "DSMS docs", + ("Materials Informatics team at Fraunhofer IWM"), + "manual", + "false", + ) +] +latex_logo = "assets/images/DSMS_logo.png" +latex_elements = {"figure_align": "H"} + +nbsphinx_allow_errors = True + +suppress_warnings = ["myst.mathjax"] diff --git a/docs/dsms.md b/docs/dsms.md new file mode 100644 index 0000000..9bb7cba --- /dev/null +++ b/docs/dsms.md @@ -0,0 +1,31 @@ +## Intoduction to DSMS + +DSMS (acronym for Dataspace Management System) by Fraunhofer IWM is a web-based Data Managment platform that manages heterogeneous data and features using semantic and analytical capabilities. + +### 1.Introduction + +DSMS platform promotes and enables the provenance and catalogization of data through a combintation of classical relational databases and semantic technologies. By enabling the interoperabilty to third party data sources, Fraunhofer IWM demonstrates this though particular use cases in material science and manufacturing in public research projects for industry 4.0. + +![DSMS_Intro](assets/images/DSMS_Mod.jpg) + + +For the DSMS, Pydantic has been used extensively. Pydantic is a Python library that leverages type annotations for data validation and settings management. It ensures that data conforms to predefined schemas by validating or serializing it at runtime and thus facilitates strict type checking and data integrity. +In next chapters we look at the two most fundamental building blocks of DSMS which is KTypes and KItems. + +### 1.1. KItems and KTypes + +**What is a KType?** + +KType stands for knowledge type and categorizes types of knowledge instances and what kind of associated details is relevant for a particular KType (as shown in the attached image). It basically describes a concept and its schema. + +**What is a KItem?** + +KItem stands for knowledge item and represents an individual instance of a KType following its property schema and functionalities. Knowledge items also capture the concepts of data containers and digital twin. + +**How is it helpful?** + +This approach streamlines the schematisation, conceptualization and structurisation of data for various domains and applications. Technically speaking, it builds an ideal base for the integration of data and knowledge into large dataspaces. KItems classified through KTypes didactically, support the upscaling of information into knowledge graphs by additional semantic annotations - which are usually mapped by ontologists. + +KItems and KTypes embody the concepts of digital twins and data containers by providing a structured and semantic framework for representing real-world materials and processes in the manufacturing industry. + +![DSMS](assets/images/DSMS.jpg) diff --git a/docs/dsms_sdk/dsms_config_schema.md b/docs/dsms_sdk/dsms_config_schema.md new file mode 100644 index 0000000..c0907b7 --- /dev/null +++ b/docs/dsms_sdk/dsms_config_schema.md @@ -0,0 +1,58 @@ +# DSMS Config Schema + +The `Configuration` class for the DSMS Python SDK is designed to handle various settings required to connect and interact with a DSMS instance. This documentation provides a detailed overview of the configurable properties, their types, defaults, and descriptions. + + +This section describes the configuration properties for the DSMS Python SDK. + +## Configuration Fields + +| Field Name | Description | Type | Default | Property Namespace | Required/Optional | +|:----------------:|:--------------------------------------------------------------------------------------------:|:--------------------:|:--------------------:|:------------------:|:-----------------:| +| Host URL | URL of the DSMS instance to connect. | AnyUrl | Not Applicable | `host_url` | Required | +| Request timeout | Timeout in seconds until the request to the DSMS is timed out. | int | `120` | `request_timeout` | Optional | +| SSL verify | Whether the SSL of the DSMS shall be verified during connection. | bool | `True` | `ssl_verify` | Optional | +| Username | User name for connecting to the DSMS instance | Optional[SecretStr] | `None` | `username` | Optional | +| Password | Password for connecting to the DSMS instance | Optional[SecretStr] | `None` | `password` | Optional | +| Token | JWT bearer token for connecting to the DSMS instance | Optional[SecretStr] | `None` | `token` | Optional | +| Ping DSMS | Check whether the host is a DSMS instance or not. | bool | `True` | `ping_dsms` | Optional | +| Encoding | General encoding to be used for reading/writing serializations. | str | “utf-8” | `encoding` | Optional | +| Datetime format | Datetime format used in the DSMS instance. | str | “%Y-%m-%dT%H:%M:%S.%f” | `datetime_format` | Optional | +| KItem repository | Repository of the triplestore for KItems in the DSMS | str | `knowledge` | `kitem_repo` | Optional | +| SPARQL Object for units | Class and Module specification in order to retrieve the units. | str | `dsms.`
`knowledge.`
`semantics.`
`units.`
`sparql:`
`UnitSparqlQuery` | `units_sparql_object`| Optional | +| Individual Slugs | When set to `True`, the slugs of the KItems will receive the first few characters of the KItem-id, when the slug is derived automatically from the KItem-name. | bool | `True` | `individual_slugs` | Optional | +| Display units | Whether the custom properties or the dataframe columns shall directly reveal their unit when printed. WARNING: This might lead to performance issues. | bool | `False` | `display_units` | Optional | +| Autocomplete units | When a unit is fetched but does not hold a symbol next to its URI, it shall be fetched from the respective ontology (which is general side effect from the `units_sparq_object`).
WARNING: This might lead to performance issues. | bool | `True` | `autocomplete_units` | Optional | +| QUDT units | URI of the QUDT unit ontology | str | `http://qudt.org/2.1/vocab/unit` | `qudt_units` | Optional | +| QUDT Quantity Kinds | URI of the QUDT quantity kind ontology | str | `http://qudt.org/vocab/quantitykind/` | `qudt_quantity_kinds` | Optional | +| Hide properties | Properties to hide while printing, e.g {'external_links'} | Set[str] | `{}` | `hide_properties` | Optional | +| Log level | Logging level | str | None | `log_level` | Optional | + +## Example Usage +```python +from dsms import DSMS + + +config = DSMS( + host_url="https://dsms.example.com", + request_timeout=30, + ssl_verify=True, + username="****", + password="****", + token=None, + ping_dsms=True, + individual_slugs=True, + encoding="utf-8", + datetime_format="%Y-%m-%dT%H:%M:%S.%f", + display_units=False, + autocomplete_units=True, + kitem_repo="knowledge-items", + qudt_units="http://qudt.org/2.1/vocab/unit", + qudt_quantity_kinds="http://qudt.org/vocab/quantitykind/", + units_sparql_object="dsms.knowledge.semantics.units.sparql:UnitSparqlQuery", + hide_properties={"external_links"}, + log_level="INFO", +) + +print(dsms.config) +``` diff --git a/docs/dsms_sdk/dsms_kitem_schema.md b/docs/dsms_sdk/dsms_kitem_schema.md new file mode 100644 index 0000000..9ab6eec --- /dev/null +++ b/docs/dsms_sdk/dsms_kitem_schema.md @@ -0,0 +1,272 @@ +# DSMS KItem Schema + +A Kitem has several properties (pydantic [`Fields`](https://docs.pydantic.dev/latest/concepts/fields/), simply referenced as `Fields` in the following) which enable it to handle data effectively. This section briefly describes the properties a Kitem can consist of, or in simple words, the schema of a KItem. + +The schema contains complex types and references, indicating an advanced usage scenario where various objects (like KItems and their properties) are interconnected. It also includes customizations like optional and default values, arrays of references, and conditional formats (e.g., UUID formats). + + +## KItem Fields + +![kitem_schema_uml](../assets/images/UML_KItem_schema.jpg) + +| Field Name | Description | Type | Default | Property Namespace | Required / Optional | +|:-----------------:|:--------------------------------------------------------------------------------------------------------:|:-------------------------------------------------:|:--------:|:------------------:|:-----------------:| +| Name | Human-readable name of the KItem. | string | Not Applicable | `name` | Required | +| ID | ID of the KItem | Union[UUID, string] | Not Applicable | `id` | Optional | +| Slug | A unique slug identifier for the KItem, minimum of 4 characters. | string | `None` | `slug` | Optional | +| Ktype ID | The type ID of the KItem | Union[Enum, string] | Not Applicable | `ktype_id` | Required | +| Created At | Timestamp of when the KItem was created. | Union[string, datetime] | `None` | `created_at` | Automatically generated | +| Updated At | Timestamp of when the KItem was updated. | Union[string, datetime] | `None` | `updated_at` | Automatically generated | +| Avatar | The avatar of the KItem. | Union[[Avatar](#avatar-fields), Dict[str, Any]] | `None` | `avatar` | Optional | +| Avatar Exists | Whether the KItem holds an avatar or not. | boolean | `False` | `avatar_exists` | Automatically generated | +| Custom Properties | A set of custom properties related to the KItem. | Any | `None` | `custom_properties`| Optional | +| Summary | A brief human-readable summary of the KItem | string | `None` | `summary` | Optional | +| KItem Apps | A list of applications associated with the KItem | List[[App](#app-fields)] | `[ ]` | `kitem_apps` | Optional | +| Annotations | A list of annotations related to the KItem | List[[Annotation](#annotation-fields)] | `[ ]` | `annotations` | Optional | +| Affiliations | A list of affiliations associated with the KItem | List[[Affiliation](#affiliation-fields)] | `[ ]` | `affiliations` | Optional | +| Contacts | Contact information related to the KItem | List[[ContactInfo](#contactinfo-fields)] | `[ ]` | `contacts` | Optional | +| External Links | A list of external links related to the KItem | List[[ExternalLink](#externallink-fields)] | `[ ]` | `external_links` | Optional | +| Attachments | A list of file attachments associated with the KItem | List [Union [[Attachment](#attachment-fields)], string] | `[ ]` | `attachments` | Optional | +| Dataframe | Dataframe associated with the KItem, e.g. a time series | Union[List[[Column](#column-fields)], pd.DataFrame, Dictionary[string, Union[List, Dictionary]]] | `None` | `dataframe` | Optional | +| Linked KItems | List of other KItems linked to this KItem | List[Union[[LinkedKItem](#linkedkitem-fields), "KItem"]] | `[ ]` | `linked_kitems` | Optional | +| User Groups | User groups with access to this KItem | List[[UserGroup](#usergroup-fields)] | `[ ]` | `user_groups` | Optional | + +### Example Usage +```python + +item = KItem( + name="Glass Bending machine 01", + slug="1234", + ktype_id="Testing Machine", + custom_properties={"location": "Room01", "max_force": "100Pa"}, + summary="This is a summary", + kitem_apps=[ + {"executable": "my_analysis_file", + "title": "Analysis", + "description": "Analysis the tensile strength from machine data"} + ], + annotations=["http://example.org/sample_kitem/annotation"], + affiliations=[ + {"name": "Institute ABC"} + ], + contacts=[ + {"name": "John Doe", "email": "john.doe@example.com"} + ], + external_links=[ + {"label": "Project Website", "url": "https://example.com"} + ], + attachments=["research_data.csv"], + linked_kitems=[another_kitem], + user_groups=[ + {"group_id": "33305", "name": "DigiMaterials"} + ] +) +``` + + +## App Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:--------:|:-------:|:------------------:|:-----------------:| +| KItem App ID | ID of the KItem App | integer | `None` | `kitem_app_id` | Automatically generated | +| Executable | Name of the executable | string | `None` | `executable` | Required | +| Title | Title of the application | string | `None` | `title` | Required | +| Description | Description of the application | string | `None` | `description` |Required | +| Tags | Tags related to the application | Dict | `None` | `tags` | `tags` | Optional | +| Additional properties | Additional properties related to the application | [Additional Properties](#additional-properties-fields) | `None` | `additional_properties` | Optional | + +### Example Usage +```python +sample_kitem.kitem_apps = [{ + "executable": "my_application", + "title": "My Application", + "description": "My Application for analysis.", +}] +``` + +## Additional Properties Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:--------:|:-------:|:------------------:|:-----------------:| +| Trigger Upon Upload | Whether the application is triggered when an attachment is uploaded | boolean | `False` | `triggerUponUpload` | Optional | +| Trigger Upon Extension | File extensions for which the upload shall be triggered | List[string] | `None` | `triggerUponUploadFileExtensions` | Optional | + +### Example Usage +```python +item.kitem_apps = [ + { + "executable": "my_yaml_file", + "title": "Data2RDF", + "additional_properties": { + "triggerUponUpload": True, + "triggerUponUploadFileExtensions": [".csv"], + }, + } + ] +``` + +## Annotation Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:--------:|:-------:|:------------------:|:-----------------:| +| IRI | IRI of the annotation | string | Not Applicable | `iri` | Required | +| Name | Name of the annotation | string | Not Applicable | `name` | Required | +| Namespace | Namespace of the annotation | string | Not Applicable | `namespace` | Required | + +### Example Usage +```python +sample_kitem.annotations = [ + "http://example.org/TensileTest" +] +``` +```python +sample_kitem.annotations = [ + { + "iri":"http://example.org/TensileTest", + "name": "TensileTest", + "namespace": "http://example.org" + } +] +``` + +## Affiliation Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:--------:|:-------:|:------------------:|:-----------------:| +| Name | Name of the affiliation | string | Not Applicable | `name` | Required | + +### Example Usage +```python +sample_kitem.affiliations = [{"name": "Research BAC"}] +``` + +## Avatar Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +| :-----------------:|:---------------------------------:|:--------:|:-------:|:------------------:|:-----------------:| +| File | The file path to the image or PIL.Image object when setting a new avatar is set | Union[string, PIL.Image] | `None` | `file` | Optional | +| Include QR code | Include QR code in the image | bool | `False` | `include_qr` | Optional | + +### Example Usage +```python +sample_kitem.avatars = [ + { + "file": "my_avatar.jpg", + "include_qr": True + } +] +``` + +## ContactInfo Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:-------------:|:-------:|:------------------:|:-----------------:| +| Email | Email of the contact person | string | Not Applicable | `email` | Required | +| Name | Name of the contact person | string | Not Applicable | `name` | Required | +| User Id | User ID of the contact person | string (UUID) | `None` | `user_id` | Optional | + +### Example Usage +```python +sample_kitem.contacts = [ + { + "email": "research.abc@gmail.com", + "name": "project01@research.abc.de", + "user_id":"33f24ee5-2f03-4874-854d-388af782c4c3" + } +] +``` + +## ExternalLink Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:--------------------------:|:-------:|:------------------:|:-----------------:| +| Label | Label of the external link | string | Not Applicable | `label` | Required | +| Url | URL of the external link | string , format: URI, minLength: 1 | Not Applicable | `url` | Required | + +### Example Usage +```python +sample_kitem.external_links = [ + { + "label": "project link", + "url": "www.projectmachine01.com" + } +] +``` + + +## Attachment Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:--------:|:-------:|:------------------:|:-----------------:| +| Name | File name of the attachment | string | Not Applicable | `name` | Required | +| Content | Content of the attachment | string | `None` | `content` | Optional | + +### Example Usage +```python +sample_kitem.attachments = ["research_data.csv"] +``` + +```python +sample_kitem.attachments = [ + { + "name": "research_data.csv", + "content": "A,B,C\n1,2,3\n4,5,6" + } +] +``` + +## Column Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:--------:|:-------:|:------------------:|:-----------------:| +| Name | Name of the column | string | Not Applicable | `name` | Required | +| Column ID | ID of the column | integer | Not Applicable | `column_id` | Required | + +### Example Usage +```python +sample_kitem.dataframe = { + "A": [1, 4], + "B": [2, 5], + "C": [3, 6] +} +``` + + +## LinkedKItem Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:-------------:|:-------:|:------------------:|:-----------------:| +| Id | ID of the KItem to be linked | string (UUID) | `None` | `id` | Required | +| Source Id | Source Id of the KItem which has been linked | string (UUID) | `None` | `source_id` | Required | + +### Example Usage +```python +sample_kitem.linked_kitems = [ + { + "id": "3e894d2c-d1a5-42ca-b6e2-cbbc09e0e686", # id of the target KItem + } +] +``` +```python +sample_kitem.linked_kitems = [ + another_kitem +] +``` + +## UserGroup Fields + +| Sub-Property Name | Description | Type | Default | Property Namespace | Required/Optional | +|:-----------------:|:---------------------------------:|:-------------:|:-------:|:------------------:|:-----------------:| +| Id | KItem ID related to the KItem property | string (UUID) | `None` | `id` | Required | +| Group Id | ID of the user group | string | `None` | `group_id` | Required | +| Name | Name of the user group | string | `None` | `name` | Required | + +### Example Usage +```python +sample_kitem.user_groups = [ + { + "group_id": "33305", + "name": "22205" + } +] +``` diff --git a/docs/dsms_sdk/dsms_sdk.md b/docs/dsms_sdk/dsms_sdk.md new file mode 100644 index 0000000..db1fe36 --- /dev/null +++ b/docs/dsms_sdk/dsms_sdk.md @@ -0,0 +1,154 @@ + +# About DSMS-SDK + +## 1. Overview + +In the dynamic world of Material Science, the introduction of our latest development brings a new way of interaction with our Dataspace Management System (DSMS): The DSMS Python SDK. + +**What is the DSMS SDK?** + +SDK stands for Software Development Kit. In our case, it is a Python-based package for the interaction with the DSMS. This means that all fundamental functionalities of the DSMS can be accessed through a Python interface now! + +**How does the SDK work?** + +Just install it on your local system via the pip command line interface: + +```bash +pip install dsms-sdk +``` + +... and start connecting to your central DSMS instance remotely, e.g. by integrating it into your own Python scripts and packages + +The SDK functionalities are listed below: +1. Managing Knowledge-Items. +2. Creating, updating and deleting meta data and properties, e.g. date, operator, material response data for a conducted tensile test. +3. Administrating authorship, contact information and supplementary information. +4. Semantic annotation of K-Items. +5. Conduct simple free-text searches and SPARQL queries. +6. Linking K-Items to other K-Items. +7. Linking Apps to K-Items, triggered, for example, during a file upload. +8. Performing simple file upload and download of file attachments. +9. Export of a knowledge (sub) graph into TTL/JSON-LD. + + +Click on the link to go to the Github repository of the Python based DSMS-SDK : [Git repo](https://github.com/MI-FraunhoferIWM/dsms-python-sdk) + +![dsms-sdk](../assets/images/DSMS_SDK.jpg) + +## 2. Installation + +### Installation and Setup Guide + +How to install and setup the dsms-python-sdk. + +### Pre-Requisites + +Before using DSMS-SDK make sure to register an account in a DSMS Instance of your choice. You could do this by sending an email to the following contacts: + +- [Yoav Nahshon](mailto:yoav.nahshon@iwm.fraunhofer.de) (Fraunhofer Institute for Mechanics of Materials IWM) +- [Matthias Büschelberger](mailto:matthias.bueschelberger@iwm.fraunhofer.de) (Fraunhofer Institute for Mechanics of Materials IWM) + +After following the above steps, you could use either of the two ways to install DSMS-SDK to your machine. + +#### Method 1: Via PyPI + +To install the [DSMS Python SDK](https://pypi.org/project/dsms-sdk/), you can use the Python Package Index (PyPI). Copy the below command to install DSMS SDK from PyPI: + +```bash +pip install dsms-sdk +``` + +#### Method 2: Cloning Github Repo + +Download and install on th8e local machine : + +```bash +git clone git@github.com:MI-FraunhoferIWM/dsms-python-sdk.git +cd dsms-python-sdk +pip install . +``` + +### Connecting to DSMS + +You need to authenticate yourself to connect with dsms using the `dsms-sdk` Python package which can be done by following one of the below given steps: + +1. **Pick the DSMS host of your choice.** + + The following are the instances of the DSMS you could choose from: + + - [StahlDigital](https://lnkd.in/gfwe9a36) + - [KupferDigital](https://lnkd.in/g8mvnM3K) + - [DiMAT](https://lnkd.in/g46baB6J) + +2. **Authentication Options** + + You can log into the DSMS core using one of the following methods: + + **a.** **Pass as Keyword Arguments** + + Directly pass your credentials (stored in a .env file) as keyword arguments when initializing the SDK. + + ```python + from dsms import DSMS + dsms = DSMS(env="../.env") + ``` + + Then add the relevant information in the .env file for setting up the login with DSMS core via SDK. Here, we choose the stahldigital instance for demo: + + ``` + DSMS_HOST_URL = https://stahldigital.materials-data.space + DSMS_USERNAME = {YOUR_USERNAME} + DSMS_PASSWORD = {YOUR_PASSWORD} + ``` + + Now save the file and the user is ready to connect with DSMS-Core via SDK. + + **b.** **Pass Config Variables while Initialization** + + Use configuration variables during initialization. Use the `getpass` module for securely entering your password. + + ```python + import getpass + from dsms import DSMS + dsms_host = "https://stahldigital.materials-data.space" + dsms_username = input("Enter your username: ") + dsms_password = getpass.getpass("Enter your password: ") + + dsms = DSMS(host=dsms_host, username=dsms_username, password=dsms_password) + ``` + + **c. Using Frontend and get token**: + + The user can login in the DSMS instance of choice and then use the login token, which can be obtained by following the below steps from the DSMS frontend interface. + + Step 1: Login into the selected dataspace instance of your choice. + + ![copy_token_1](../assets/images/copy_token_1.jpg) + + Step 2: Enter your credentials + + ![copy_token_2](../assets/images/copy_token_2.jpg) + + Step 3: After logging in you will land up on the home page. Now proceed to the my profile option + + ![copy_token_3](../assets/images/copy_token_3.jpg) + + Step 4: The my profile option should look something like below. Now click on Advanced. + + ![copy_token_4](../assets/images/copy_token_4.jpg) + + Step 5: After landing up on Advanced section, click on the copy token to clipboard + + ![copy_token_6](../assets/images/copy_token_5.jpg) + + Step 6: Now the login token is in the clipboard. Now paste the copied login token from the frontend in the DSMS_TOKEN attribute of the .env file + ``` + DSMS_HOST_URL = https://stahldigital.materials-data.space + DSMS_TOKEN = {YOUR_COPIED_TOKEN} + ``` + +Now you are ready to use dsms-sdk. Do check out the tutorials section to try out some basic examples on how to use dsms-sdk. + +The next sections covers about the schema of fundamental classes crucial for users to know about DSMS when using the platform. Below given explains about the Schema of `KItem` and its associated properties in DSMS. + +Now the next section gives a brief explanation about the Schema of `Config` class of DSMS and its associated properties in DSMS. diff --git a/docs/dsms_sdk/tutorials/1_introduction.ipynb b/docs/dsms_sdk/tutorials/1_introduction.ipynb new file mode 100644 index 0000000..b9bcb50 --- /dev/null +++ b/docs/dsms_sdk/tutorials/1_introduction.ipynb @@ -0,0 +1,141 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 1. Connecting with the SDK to DSMS\n", + "\n", + "In this tutorial we see the overview on how to setup and basic use DSMS-SDK\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.1. Setting up\n", + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", + "\n", + "Now let us import the needed classes and functions for this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KItem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\".env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2. Introduction to KItems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see which kind of DSMS-object we own as a user (in the beginning, we own none):" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dsms.kitems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can investigate what a KItem needs in order to be created. KItems are entirely based on Pydantic-Models (v2), hence the properties (in Pydantic called Fields) are automatically validated once we set them.\n", + "\n", + "The schema of the KItem itself is a JSON schema which is machine-readable and can be directly incorporated into Swagger-supported APIs like e.g. FastAPI.\n", + "\n", + "We can investigate the KTypes defined in the remote instance:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KTypes.Organization\n", + "KTypes.App\n", + "KTypes.Dataset\n", + "KTypes.DatasetCatalog\n", + "KTypes.Expert\n", + "KTypes.Test\n", + "KTypes.Specimen\n", + "KTypes.Batch\n", + "KTypes.Resource\n", + "KTypes.TestingMachine\n" + ] + } + ], + "source": [ + "for ktype in dsms.ktypes:\n", + " print(ktype)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/dsms_sdk/tutorials/2_creation.ipynb b/docs/dsms_sdk/tutorials/2_creation.ipynb new file mode 100644 index 0000000..63c950d --- /dev/null +++ b/docs/dsms_sdk/tutorials/2_creation.ipynb @@ -0,0 +1,333 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 2. Create KItems with the SDK\n", + "\n", + "In this tutorial we see how to create new Kitems." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1. Setting up\n", + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", + "\n", + "Now let us import the needed classes and functions for this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KItem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\".env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "### 2.2: Create KItems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can make new KItems by simple class-initiation: (Make sure existing KItems are not given as input). \n", + "#" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = KTypes.TestingMachine, \n", + "\n", + "\tin_backend = False, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = None, \n", + "\n", + "\tupdated_at = None, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: TestingLab GmBH, \n", + "\t\tLocation: A404, \n", + "\t\tModel Number: Bending Test Machine No 777\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item = KItem(\n", + " name=\"Machine-1\",\n", + " ktype_id=dsms.ktypes.TestingMachine,\n", + " custom_properties={\"Producer\": \"TestingLab GmBH\",\n", + " \"Location\": \"A404\",\n", + " \"Model Number\" : \"Bending Test Machine No 777\"\n", + " },\n", + ")\n", + "\n", + "item" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Remember: changes are only syncronized with the DSMS when you call the `commit`-method:" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'https://bue.materials-data.space/knowledge/testing-machine/machine-1-dd091666'" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dsms.commit()\n", + "item.url" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, the object we created before running the `commit`-method has automatically been updated, e.g. with the creation- and update-timestamp. We can check this with the below command:" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\tupdated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: TestingLab GmBH, \n", + "\t\tLocation: A404, \n", + "\t\tModel Number: Bending Test Machine No 777\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To just get the name of the item, we can do it as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Machine-1'" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item.name" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To check the type of the item newly created we can use the following:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dsms.knowledge.kitem.KItem" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "type(item)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Now you can check if the particular kitem is in the list of KItems. This can be done either by using the command:\n", + " `\n", + " dsms.kitems\n", + " `\n", + " or by logging into the frontend dsms instance." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/dsms_sdk/tutorials/3_updation.ipynb b/docs/dsms_sdk/tutorials/3_updation.ipynb new file mode 100644 index 0000000..cfa9245 --- /dev/null +++ b/docs/dsms_sdk/tutorials/3_updation.ipynb @@ -0,0 +1,292 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 3. Updating KItems with the SDK\n", + "\n", + "In this tutorial we see how to update existing Kitems." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.1. Setting up\n", + "\n", + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", + "\n", + "\n", + "Now let us import the needed classes and functions for this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KItem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\".env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now lets get the kitem we created in the [2nd tutorial : Creation of Kitems](2_creation.ipynb)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Machine-1\n" + ] + } + ], + "source": [ + "item = dsms.kitems[-1]\n", + "print(item.name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.2. Updating Kitems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, we would like to update the properties of our KItem we created previously.\n", + "\n", + "Depending on the schema of each property (see [DSMS KItem Schema](../dsms_kitem_schema.md)), we can simply use the standard `list`-method as we know them from basic Python (e.g. for the `annotations`, `attachments`, `external_link`, etc). \n", + "\n", + "Other properties which are not `list`-like can be simply set by attribute-assignment (e.g. `name`, `slug`, `ktype_id`, etc)." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "item.name = \"Machine-1\"\n", + "item.custom_properties.Producer = \"Machinery GmBH\"\n", + "item.attachments.append(\"testfile.txt\")\n", + "item.annotations.append(\"www.machinery.org/\")\n", + "item.external_links.append(\n", + " {\"url\": \"http://machine.org\", \"label\": \"machine-link\"}\n", + ")\n", + "item.contacts.append({\"name\": \"machinesupport\", \"email\": \"machinesupport@group.mail\"})\n", + "item.affiliations.append(\"machine-team\")\n", + "item.user_groups.append({\"name\": \"machinegroup\", \"group_id\": \"123\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see now that the local system path of the attachment is changed to a simply file name, which means that the upload was successful. If not so, an error would have been thrown during the `commit`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see the updates when we print the item:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [\n", + "\t\t{\n", + "\t\t\tiri: www.machinery.org/,\n", + "\t\t\tname: ,\n", + "\t\t\tnamespace: www.machinery.org,\n", + "\t\t\tdescription: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tattachments = [\n", + "\t\t{\n", + "\t\t\tname: testfile.txt\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [\n", + "\t\t{\n", + "\t\t\tname: machine-team\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [\n", + "\t\t{\n", + "\t\t\tname: machinesupport,\n", + "\t\t\temail: machinesupport@group.mail,\n", + "\t\t\tuser_id: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcreated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\tupdated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\texternal_links = [\n", + "\t\t{\n", + "\t\t\tlabel: machine-link,\n", + "\t\t\turl: http://machine.org/\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [\n", + "\t\t{\n", + "\t\t\tname: machinegroup,\n", + "\t\t\tgroup_id: 123\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: Machinery GmBH, \n", + "\t\tLocation: A404, \n", + "\t\tModel Number: Bending Test Machine No 777\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Furthermore we can also download the file we uploaded again:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\t\t\t Downloaded file: {\n", + "\t\t\tname: testfile.txt\n", + "\t\t}\n", + "|------------------------------------Beginning of file------------------------------------|\n", + "This is a calibration protocol!\n", + "|---------------------------------------End of file---------------------------------------|\n" + ] + } + ], + "source": [ + "for file in item.attachments:\n", + " download = file.download()\n", + "\n", + " print(\"\\t\\t\\t Downloaded file:\", file)\n", + " print(\"|------------------------------------Beginning of file------------------------------------|\")\n", + " print(download)\n", + " print(\"|---------------------------------------End of file---------------------------------------|\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/dsms_sdk/tutorials/4_deletion.ipynb b/docs/dsms_sdk/tutorials/4_deletion.ipynb new file mode 100644 index 0000000..fe44b17 --- /dev/null +++ b/docs/dsms_sdk/tutorials/4_deletion.ipynb @@ -0,0 +1,408 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 4. Deleting KItems with the SDK\n", + "\n", + "In this tutorial we see how to delete new Kitems and their properties." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.1. Setting up\n", + "\n", + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", + "\n", + "Now let us import the needed classes and functions for this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KItem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\".env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then lets see the Kitem we are interested in to remove." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [\n", + "\t\t{\n", + "\t\t\tiri: www.machinery.org/,\n", + "\t\t\tname: ,\n", + "\t\t\tnamespace: www.machinery.org,\n", + "\t\t\tdescription: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tattachments = [\n", + "\t\t{\n", + "\t\t\tname: testfile.txt\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [\n", + "\t\t{\n", + "\t\t\tname: machine-team\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [\n", + "\t\t{\n", + "\t\t\tname: machinesupport,\n", + "\t\t\temail: machinesupport@group.mail,\n", + "\t\t\tuser_id: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcreated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\tupdated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\texternal_links = [\n", + "\t\t{\n", + "\t\t\tlabel: machine-link,\n", + "\t\t\turl: http://machine.org/\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [\n", + "\t\t{\n", + "\t\t\tname: machinegroup,\n", + "\t\t\tgroup_id: 123\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: Machinery GmBH, \n", + "\t\tLocation: A404, \n", + "\t\tModel Number: Bending Test Machine No 777\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")\n" + ] + } + ], + "source": [ + "item = dsms.kitems[-1]\n", + "print(item)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.2. Deletion of KItems and their properties" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also remove properties from the KItem without deleting the KItem itself.\n", + "\n", + "For the `list`-like properties, we can use the standard `list`-methods from basic Python again (e.g. `pop`, `remove`, etc. or the `del`-operator).\n", + "\n", + "For the other, non-`list`-like properties, we can simply use the attribute-assignment again.\n", + "\n", + "When we only want single parts of the properties in the KItem, we can do it like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{\n", + "\t\t\tname: machinegroup,\n", + "\t\t\tgroup_id: 123\n", + "\t\t}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item.attachments.pop(0)\n", + "item.annotations.pop(0)\n", + "item.external_links.pop(0)\n", + "item.contacts.pop(0)\n", + "item.user_groups.pop(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, we can also reset the entire property by setting it to e.g. an empty list again:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "item.affiliations = []" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can delete the custom properties by setting the property to an empty dict:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "item.custom_properties = {}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Send the changes to the DSMS with the `commit`-method:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "See the changes:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [\n", + "\t\t{\n", + "\t\t\tname: machine-team\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [\n", + "\t\t{\n", + "\t\t\tname: machinesupport,\n", + "\t\t\temail: machinesupport@group.mail,\n", + "\t\t\tuser_id: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcreated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\tupdated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\texternal_links = [\n", + "\t\t{\n", + "\t\t\tlabel: machine-link,\n", + "\t\t\turl: http://machine.org/\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tid: dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\t\tcontent: {}\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, we can also delete the whole KItem from the DSMS by applying the `del`-operator to the `dsms`-object with the individual `KItem`-object:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "del dsms[item]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Commit the changes:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now to check if the particular kitem was removed, we can do this by using the command:\n", + " `\n", + " dsms.kitems\n", + " `\n", + " or by logging into the frontend dsms instance." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/dsms_sdk/tutorials/5_search.ipynb b/docs/dsms_sdk/tutorials/5_search.ipynb new file mode 100644 index 0000000..9d54b87 --- /dev/null +++ b/docs/dsms_sdk/tutorials/5_search.ipynb @@ -0,0 +1,485 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 5. Searching KItems with the SDK\n", + "\n", + "In this tutorial we see how to search existing Kitems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.1. Setting up\n", + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", + "\n", + "\n", + "Now let us import the needed classes and functions for this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KItem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\".env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.2. Searching for KItems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this section, we would like to search for specfic KItems we created in the DSMS.\n", + "\n", + "For this purpose, we will firstly create some KItems and apply the `search`-method on the `DSMS`-object later on in order to find them again in the DSMS.\n", + "\n", + "We also want to demonstrate here, that we can link KItems to each other in order to find e.g. a related item of type `DatasetCatalog`. For this strategy, we are using the `linked_kitems`- attribute and the `id` of the item which we would like to link.\n", + "\n", + "The procedure looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "item1 = KItem(\n", + " name=\"Machine-1\",\n", + " ktype_id=dsms.ktypes.TestingMachine,\n", + " custom_properties={\"Producer\": \"TestingLab GmBH\",\n", + " \"Room Number\": \"A404\",\n", + " \"Description\": \"Bending Test Machine\"\n", + " }\n", + ")\n", + "\n", + "item2 = KItem(\n", + " name=\"Machine-2\",\n", + " ktype_id=dsms.ktypes.TestingMachine,\n", + " custom_properties={\"Producer\": \"StressStrain GmBH\",\n", + " \"Room Number\": \"B500\",\n", + " \"Description\": \"Compression Test Machine\"\n", + " }\n", + ")\n", + "\n", + "item3 = KItem(\n", + " name=\"Specimen-1\", \n", + " ktype_id=dsms.ktypes.Specimen,\n", + " linked_kitems=[item1],\n", + " custom_properties={\"Geometry\": \"Cylindrical 150mm x 20mm x 40mm\",\n", + " \"Material\": \"Concrete\",\n", + " \"Project ID\": \"ConstructionProject2024\"\n", + " }\n", + "\n", + ")\n", + "item4 = KItem(\n", + " name=\"Specimen-2\",\n", + " ktype_id=dsms.ktypes.Specimen,\n", + " linked_kitems=[item2],\n", + " custom_properties={\"Geometry\": \"Rectangular 200mm x 30mm x 20mm\",\n", + " \"Material\": \"Metal\",\n", + " \"Project ID\": \"MetalBlenders2024\"\n", + " }\n", + ")\n", + "\n", + "item5 = KItem(\n", + " name=\"Research Institute ABC\",\n", + " ktype_id=dsms.ktypes.Organization,\n", + " linked_kitems=[item1],\n", + " annotations=[\n", + " {\n", + " \"iri\": \"www.researchBACiri.org/foo\",\n", + " \"name\": \"research ABC Institute\",\n", + " \"namespace\": \"research\",\n", + " }\n", + " ],\n", + ")\n", + "\n", + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "####

Note : Here in this tutorial, we use dsms.search with `limit=1` to maintain readability but the user can adjust the variable `limit` as per requirement.

\n", + "\n", + "\n", + "Now, we are apply to search for e.g. kitems of type `TestingMachine`:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = 0f49b0ad-2a98-4c10-8440-fd3c37363646, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-0f49b0ad, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [\n", + "\t\t\n", + "\t\t\tid: 075709da-5481-4cb4-a8ee-c18db34ce9d3\n", + "\t\t\tname: Specimen-1\n", + "\t\t\tslug: specimen-1-075709da\n", + "\t\t\tktype_id: specimen\n", + "\t\t\tsummary: None\n", + "\t\t\tavatar_exists: False\n", + "\t\t\tannotations: []\n", + "\t\t\tlinked_kitems: [{\n", + "\t\t\tid: 0f49b0ad-2a98-4c10-8440-fd3c37363646\n", + "\t\t}]\n", + "\t\t\texternal_links: []\n", + "\t\t\tcontacts: []\n", + "\t\t\tauthors: [{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}]\n", + "\t\t\tlinked_affiliations: []\n", + "\t\t\tattachments: []\n", + "\t\t\tuser_groups: []\n", + "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", + "\t\t\tcreated_at: 2024-08-19T18:26:00.499708\n", + "\t\t\tupdated_at: 2024-08-19T18:26:00.499708\n", + "\t\t\n", + "\t], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-19 18:26:00.247787, \n", + "\n", + "\tupdated_at = 2024-08-19 18:26:00.247787, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: TestingLab GmBH, \n", + "\t\tRoom Number: A404, \n", + "\t\tDescription: Bending Test Machine\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")\n", + "\n", + "\n" + ] + } + ], + "source": [ + "for result in dsms.search(ktypes=[dsms.ktypes.TestingMachine], limit=1):\n", + " print(result.hit)\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... and for all of type `Organization` and `DatasetCatalog`:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = Research Institute ABC, \n", + "\n", + "\tid = 21aa50c3-5ec2-4ac3-aba8-69071a4287e2, \n", + "\n", + "\tktype_id = organization, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = researchinstituteabc-21aa50c3, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-19 18:26:00.740761, \n", + "\n", + "\tupdated_at = 2024-08-19 18:26:00.740761, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = None, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")\n", + "fuzziness: False\n", + "\n", + "\n" + ] + } + ], + "source": [ + "for result in dsms.search(ktypes=[dsms.ktypes.Organization, dsms.ktypes.DatasetCatalog], limit=1):\n", + " print(result.hit)\n", + " print(\"fuzziness: \", result.fuzzy)\n", + " print(\"\\n\")\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... or for all of type `Dataset` with `Specimen-1` in the name:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "for result in dsms.search(query=\"Specimen-1\", ktypes=[dsms.ktypes.Dataset], limit=1):\n", + " print(result.hit)\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... and for all of type `Organization` with the annotation `www.researchBACiri.org/foo`:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = Research Institute ABC, \n", + "\n", + "\tid = 84c089e6-feab-4ba6-a934-527594e504eb, \n", + "\n", + "\tktype_id = organization, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = researchinstituteabc-84c089e6, \n", + "\n", + "\tannotations = [\n", + "\t\t{\n", + "\t\t\tiri: www.researchBACiri.org/foo,\n", + "\t\t\tname: research ABC Institute,\n", + "\t\t\tnamespace: research,\n", + "\t\t\tdescription: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [\n", + "\t\t\n", + "\t\t\tid: 694561e9-df15-4c71-ae71-abd91df3ec23\n", + "\t\t\tname: Machine-1\n", + "\t\t\tslug: machine-1-694561e9\n", + "\t\t\tktype_id: testing-machine\n", + "\t\t\tsummary: None\n", + "\t\t\tavatar_exists: False\n", + "\t\t\tannotations: []\n", + "\t\t\tlinked_kitems: [{\n", + "\t\t\tid: 83672cdb-7584-4dd7-9b6e-d10574f1adf7\n", + "\t\t}, {\n", + "\t\t\tid: 84c089e6-feab-4ba6-a934-527594e504eb\n", + "\t\t}]\n", + "\t\t\texternal_links: []\n", + "\t\t\tcontacts: []\n", + "\t\t\tauthors: [{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}]\n", + "\t\t\tlinked_affiliations: []\n", + "\t\t\tattachments: []\n", + "\t\t\tuser_groups: []\n", + "\t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", + "\t\t\tcreated_at: 2024-08-19T18:26:08.898435\n", + "\t\t\tupdated_at: 2024-08-19T18:26:08.898435\n", + "\t\t\n", + "\t], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-19 18:26:09.390800, \n", + "\n", + "\tupdated_at = 2024-08-19 18:26:09.390800, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = None, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")\n", + "\n", + "\n" + ] + } + ], + "source": [ + "for result in dsms.search(\n", + " ktypes=[dsms.ktypes.Organization], annotations=[\"www.researchBACiri.org/foo\"], limit=1\n", + " ):\n", + " print(result.hit)\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Clean up the DSMS from the tutortial:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "del dsms[item1]\n", + "del dsms[item2]\n", + "del dsms[item3]\n", + "del dsms[item4]\n", + "del dsms[item5]\n", + "\n", + "dsms.commit()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/dsms_sdk/tutorials/6_apps.ipynb b/docs/dsms_sdk/tutorials/6_apps.ipynb new file mode 100644 index 0000000..b80dc22 --- /dev/null +++ b/docs/dsms_sdk/tutorials/6_apps.ipynb @@ -0,0 +1,1073 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 6. DSMS Apps and Pipelines\n", + "\n", + "In this tutorial we see how to create apps and run them manually" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.1: Setting up\n", + "\n", + "Before you run this tutorial: make sure to have access to a DSMS-instance of your interest, alongwith with installation of this package and have establised access to the DSMS through DSMS-SDK (refer to [Connecting to DSMS](../dsms_sdk.md#connecting-to-dsms))\n", + "\n", + "\n", + "Now let us import the needed classes and functions for this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KItem, AppConfig\n", + "import time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\".env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.1. Investigating Available Apps" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can investigate which apps are available:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[AppConfig(name=ckan-fetch, specification={'metadata': {'generateName': 'ckan-resource-request-'}}),\n", + " AppConfig(name=csv_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=csv_tensile_test_f2, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=excel_notched_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=excel_shear_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=excel_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=ternary-plot, specification={'metadata': {'generateName': 'ckan-tenary-app-'}}),\n", + " AppConfig(name=testapp2, specification={'metadata': {'generateName': 'data2rdf-'}})]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dsms.app_configs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.2 Create a new app config and apply it to a KItem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.2.1 Arbitrary python code" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To be defined." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.2.2 - Data2RDF" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 6.2.2.1 Prepare app and its config" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the following example, we would like to upload some csv with some arbitrary data and describe it through an RDF. This will give us the opportunity to harmonize the entities of the data file through ontological concepts and allow us to convert values of the data frame columns in to any compatible unit we would like to have.\n", + "\n", + "Fist of all, let us define the data:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "data = \"\"\"A,B,C\n", + "1.2,1.3,1.5\n", + "1.7,1.8,1.9\n", + "2.0,2.1,2.3\n", + "2.5,2.6,2.8\n", + "3.0,3.2,3.4\n", + "3.6,3.7,3.9\n", + "4.1,4.3,4.4\n", + "4.7,4.8,5.0\n", + "5.2,5.3,5.5\n", + "5.8,6.0,6.1\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will also give the config a defined name:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "configname = \"testapp2\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As a next step, we want to create a new app specification. The specification is following the definition of an [**Argo Workflow**](https://argo-workflows.readthedocs.io/en/latest/). The workflow shall trigger a pipeline from a docker image with the [**Data2RDF package**](https://data2rdf.readthedocs.io/en/latest/). \n", + "\n", + "The image has already been deployed on the k8s cluster of the DSMS and the workflow template with the name `dsms-data2rdf` has been implemented previously. Hence we only need to configure our pipeline for our data shown above, which we would like to upload and describe through an RDF.\n", + "\n", + "For more details about the data2rdf package, please refer to the documentation of Data2RDF mentioned above.\n", + "\n", + "The parameters of the app config are defining the inputs for our Data2RDF pipeline. This e.g. are: \n", + "\n", + "* the parser kind (`csv` here)\n", + "* the time series header length (`1` here)\n", + "* the metadata length (`0` here)\n", + "* the time series separator (`,` here)\n", + "* the log level (`DEBUG` here)\n", + "* the mapping \n", + " * `A` is the test time and has a unit in seconds\n", + " * `B` is the standard force in kilonewtons\n", + " * `C` is the absolut cross head travel in millimeters" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "parameters = [\n", + " {\"name\": \"parser\", \"value\": \"csv\"},\n", + " {\"name\": \"time_series_header_length\", \"value\": 1},\n", + " {\"name\": \"metadata_length\", \"value\": 0},\n", + " {\"name\": \"time_series_sep\", \"value\": \",\"},\n", + " {\"name\": \"log_level\", \"value\": \"DEBUG\"},\n", + " {\n", + " \"name\": \"mapping\",\n", + " \"value\": \"\"\"\n", + " [\n", + " {\n", + " \"key\": \"A\",\n", + " \"iri\": \"https://w3id.org/steel/ProcessOntology/TestTime\",\n", + " \"unit\": \"s\"\n", + " },\n", + " {\n", + " \"key\": \"B\",\n", + " \"iri\": \"https://w3id.org/steel/ProcessOntology/StandardForce\",\n", + " \"unit\": \"kN\"\n", + " },\n", + " {\n", + " \"key\": \"C\",\n", + " \"iri\": \"https://w3id.org/steel/ProcessOntology/AbsoluteCrossheadTravel\",\n", + " \"unit\": \"mm\"\n", + " }\n", + " ]\n", + " \"\"\",\n", + " },\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we add the parameters to our app specification. We assign a prefix `datardf-` which shall generate a new name with some random characters as suffix. The workflow template with the Docker image we want to run is called `dsms-data2rdf` and its `entrypoint` is `execute_pipeline`." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Define app specification\n", + "specification = {\n", + " \"apiVersion\": \"argoproj.io/v1alpha1\",\n", + " \"kind\": \"Workflow\",\n", + " \"metadata\": {\"generateName\": \"data2rdf-\"},\n", + " \"spec\": {\n", + " \"entrypoint\": \"execute_pipeline\",\n", + " \"workflowTemplateRef\": {\"name\": \"dsms-data2rdf\"},\n", + " \"arguments\": {\"parameters\": parameters},\n", + " },\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we instanciate the new app config:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "appspec = AppConfig(\n", + " name=configname,\n", + " specification=specification, # this can also be a file path to a yaml file instead of a dict\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We commit the new app config:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we would like to apply the app config to a KItem. The set the `triggerUponUpload` must be set to `True` so that the app is triggered automatically when we upload an attachment.\n", + "\n", + "Additionally, we must tell the file extension for which the upload shall be triggered. Here it is `.csv`.\n", + "\n", + "We also want to generate a qr code as avatar for the KItem with `avatar={\"include_qr\": True}`." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "item = KItem(\n", + " name=\"my tensile test experiment\",\n", + " ktype_id=dsms.ktypes.Dataset,\n", + " kitem_apps=[\n", + " {\n", + " \"executable\": appspec.name,\n", + " \"title\": \"data2rdf\",\n", + " \"additional_properties\": {\n", + " \"triggerUponUpload\": True,\n", + " \"triggerUponUploadFileExtensions\": [\".csv\"],\n", + " },\n", + " }\n", + " ],\n", + " avatar={\"include_qr\": True},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We commit the KItem:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we add our data with our attachment:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "item.attachments = [{\"name\": \"dummy_data.csv\", \"content\": data}]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And we commit again:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 6.2.2.2 Get results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can verify that the data extraction was successful:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = my tensile test experiment, \n", + "\n", + "\tid = 047c3e0e-0c4c-4815-b041-5251c2ef0882, \n", + "\n", + "\tktype_id = dataset, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = mytensiletestexperiment-047c3e0e, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [\n", + "\t\t{\n", + "\t\t\tname: dummy_data.csv\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = True, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-19 19:27:11.826889, \n", + "\n", + "\tupdated_at = 2024-08-19 19:27:11.826889, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [\n", + "\t\t{\n", + "\t\t\tkitem_app_id: 21,\n", + "\t\t\texecutable: testapp2,\n", + "\t\t\ttitle: data2rdf,\n", + "\t\t\tdescription: None,\n", + "\t\t\ttags: None,\n", + "\t\t\tadditional_properties: {triggerUponUpload: True, triggerUponUploadFileExtensions: ['.csv']}\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = None, \n", + "\n", + "\tdataframe = [\n", + "\t\t{\n", + "\t\t\tcolumn_id: 0,\n", + "\t\t\tname: TestTime\n", + "\t\t}, \n", + "\t\t{\n", + "\t\t\tcolumn_id: 1,\n", + "\t\t\tname: StandardForce\n", + "\t\t}, \n", + "\t\t{\n", + "\t\t\tcolumn_id: 2,\n", + "\t\t\tname: AbsoluteCrossheadTravel\n", + "\t\t}\n", + "\t], \n", + "\n", + "\trdf_exists = True\n", + ")\n" + ] + } + ], + "source": [ + "print(item)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And also that the RDF generation was successful:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@prefix csvw: .\n", + "@prefix dcat: .\n", + "@prefix dcterms: .\n", + "@prefix ns1: .\n", + "@prefix ns2: .\n", + "@prefix rdfs: .\n", + "@prefix xsd: .\n", + "\n", + " a dcat:Dataset ;\n", + " dcterms:hasPart ;\n", + " dcat:distribution [ a dcat:Distribution ;\n", + " dcat:accessURL \"https://bue.materials-data.space/api/knowledge/data_api/047c3e0e-0c4c-4815-b041-5251c2ef0882\"^^xsd:anyURI ;\n", + " dcat:mediaType \"http://www.iana.org/assignments/media-types/text/csv\"^^xsd:anyURI ] .\n", + "\n", + " a ;\n", + " ns1:hasUnit \"http://qudt.org/vocab/unit/MilliM\"^^xsd:anyURI .\n", + "\n", + " a ;\n", + " ns1:hasUnit \"http://qudt.org/vocab/unit/KiloN\"^^xsd:anyURI .\n", + "\n", + " a ;\n", + " ns1:hasUnit \"http://qudt.org/vocab/unit/SEC\"^^xsd:anyURI .\n", + "\n", + " a csvw:TableGroup ;\n", + " csvw:table [ a csvw:Table ;\n", + " rdfs:label \"Time series data\" ;\n", + " csvw:tableSchema [ a csvw:Schema ;\n", + " csvw:column [ a csvw:Column ;\n", + " ns1:quantity ;\n", + " csvw:titles \"C\"^^xsd:string ;\n", + " ns2:page [ a ns2:Document ;\n", + " dcterms:format \"https://www.iana.org/assignments/media-types/application/json\"^^xsd:anyURI ;\n", + " dcterms:identifier \"https://bue.materials-data.space/api/knowledge/data_api/column-2\"^^xsd:anyURI ;\n", + " dcterms:type \"http://purl.org/dc/terms/Dataset\"^^xsd:anyURI ] ],\n", + " [ a csvw:Column ;\n", + " ns1:quantity ;\n", + " csvw:titles \"A\"^^xsd:string ;\n", + " ns2:page [ a ns2:Document ;\n", + " dcterms:format \"https://www.iana.org/assignments/media-types/application/json\"^^xsd:anyURI ;\n", + " dcterms:identifier \"https://bue.materials-data.space/api/knowledge/data_api/column-0\"^^xsd:anyURI ;\n", + " dcterms:type \"http://purl.org/dc/terms/Dataset\"^^xsd:anyURI ] ],\n", + " [ a csvw:Column ;\n", + " ns1:quantity ;\n", + " csvw:titles \"B\"^^xsd:string ;\n", + " ns2:page [ a ns2:Document ;\n", + " dcterms:format \"https://www.iana.org/assignments/media-types/application/json\"^^xsd:anyURI ;\n", + " dcterms:identifier \"https://bue.materials-data.space/api/knowledge/data_api/column-1\"^^xsd:anyURI ;\n", + " dcterms:type \"http://purl.org/dc/terms/Dataset\"^^xsd:anyURI ] ] ] ] .\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(item.subgraph.serialize())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And now we are able to convert our data into any compatiable unit we want. For the `StandardForce`, it was previously `kN`, but we want to have it in `N` now:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[1300.0,\n", + " 1800.0,\n", + " 2100.0,\n", + " 2600.0,\n", + " 3200.0,\n", + " 3700.0,\n", + " 4300.0,\n", + " 4800.0,\n", + " 5300.0,\n", + " 6000.0]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item.dataframe.StandardForce.convert_to(\"N\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 6.2.2.3 Manipulate dataframe" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to retrieve the dataframe as pd.DataFrame:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TestTimeStandardForceAbsoluteCrossheadTravel
01.21.31.5
11.71.81.9
22.02.12.3
32.52.62.8
43.03.23.4
53.63.73.9
64.14.34.4
74.74.85.0
85.25.35.5
95.86.06.1
\n", + "
" + ], + "text/plain": [ + " TestTime StandardForce AbsoluteCrossheadTravel\n", + "0 1.2 1.3 1.5\n", + "1 1.7 1.8 1.9\n", + "2 2.0 2.1 2.3\n", + "3 2.5 2.6 2.8\n", + "4 3.0 3.2 3.4\n", + "5 3.6 3.7 3.9\n", + "6 4.1 4.3 4.4\n", + "7 4.7 4.8 5.0\n", + "8 5.2 5.3 5.5\n", + "9 5.8 6.0 6.1" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item.dataframe.to_df()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to overwrite the dataframe with new data:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "item.dataframe = {\n", + " \"TestTime\": list(range(100)),\n", + " \"StandardForce\": list(range(1,101)),\n", + " \"AbsoluteCrossheadTravel\": list(range(2,102))\n", + "}\n", + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to retrieve the data colum-wise:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "column: TestTime ,\n", + " data: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]\n", + "column: StandardForce ,\n", + " data: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100]\n", + "column: AbsoluteCrossheadTravel ,\n", + " data: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101]\n" + ] + } + ], + "source": [ + "for column in item.dataframe:\n", + " print(\"column:\", column.name, \",\\n\", \"data:\", column.get())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... and also to modify the dataframe directly as we need:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "new_df = item.dataframe.to_df().drop(['TestTime'], axis=1)\n", + "item.dataframe = new_df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 6.2.2.4 Run app on demand" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to run the app on demand, not being triggered automatically during the upload of an attachment every time. For this purpose, we just need to refer to the name of the app we assigned during the KItem creation ( here it is simply `data2rdf`).\n", + "\n", + "Additionally, we need to tell the `attachment_name` and hand over the access token and host url to the app by explicitly setting `set_token` and `set_host_url` to `True`.\n", + "\n", + "The app is running synchronously, hence the `job` is created when the pipeline run finished." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "job = item.kitem_apps.by_title[\"data2rdf\"].run(\n", + " attachment_name=item.attachments[0].name,\n", + " set_token=True,\n", + " set_host_url=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to retrieve the job status:" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "JobStatus(phase='Succeeded', estimated_duration=None, finished_at='08/19/2024, 19:28:06', started_at='08/19/2024, 19:27:43', message=None, progress='1/1')" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job.status" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... and the job logs:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\"[2024-08-19 19:27:49,060 - dsms_data2rdf.main - INFO]: Fetch KItem: \\n KItem(\\n\\n\\tname = my tensile test experiment, \\n\\n\\tid = 047c3e0e-0c4c-4815-b041-5251c2ef0882, \\n\\n\\tktype_id = dataset, \\n\\n\\tin_backend = True, \\n\\n\\tslug = mytensiletestexperiment-047c3e0e, \\n\\n\\tannotations = [], \\n\\n\\tattachments = [\\n\\t\\t{\\n\\t\\t\\tname: dummy_data.csv\\n\\t\\t}\\n\\t], \\n\\n\\tlinked_kitems = [], \\n\\n\\taffiliations = [], \\n\\n\\tauthors = [\\n\\t\\t{\\n\\t\\t\\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\\n\\t\\t}\\n\\t], \\n\\n\\tavatar_exists = True, \\n\\n\\tcontacts = [], \\n\\n\\tcreated_at = 2024-08-19 19:27:11.826889, \\n\\n\\tupdated_at = 2024-08-19 19:27:11.826889, \\n\\n\\texternal_links = [], \\n\\n\\tkitem_apps = [\\n\\t\\t{\\n\\t\\t\\tkitem_app_id: 21,\\n\\t\\t\\texecutable: testapp2,\\n\\t\\t\\ttitle: data2rdf,\\n\\t\\t\\tdescription: None,\\n\\t\\t\\ttags: None,\\n\\t\\t\\tadditional_properties: {triggerUponUpload: True, triggerUponUploadFileExtensions: ['.csv']}\\n\\t\\t}\\n\\t], \\n\\n\\tsummary = None, \\n\\n\\tuser_groups = [], \\n\\n\\tcustom_properties = None, \\n\\n\\tdataframe = [\\n\\t\\t{\\n\\t\\t\\tcolumn_id: 0,\\n\\t\\t\\tname: TestTime\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 1,\\n\\t\\t\\tname: StandardForce\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 2,\\n\\t\\t\\tname: AbsoluteCrossheadTravel\\n\\t\\t}\\n\\t], \\n\\n\\trdf_exists = True\\n)\\n[2024-08-19 19:27:49,073 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser arguments: {'metadata_sep': ',', 'metadata_length': '0', 'time_series_sep': ',', 'time_series_header_length': '1', 'drop_na': 'false', 'fillna': ''}\\n[2024-08-19 19:27:49,073 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser: Parser.csv\\n[2024-08-19 19:27:49,073 - dsms_data2rdf.main - INFO]: Run pipeline with the following config: {'base_iri': 'https://bue.materials-data.space/047c3e0e-0c4c-4815-b041-5251c2ef0882', 'data_download_uri': 'https://bue.materials-data.space/api/knowledge/data_api/047c3e0e-0c4c-4815-b041-5251c2ef0882', 'graph_identifier': 'https://bue.materials-data.space/047c3e0e-0c4c-4815-b041-5251c2ef0882', 'separator': '/', 'encoding': 'utf-8'}\\n[2024-08-19 19:27:54,079 - dsms_data2rdf.main - INFO]: Pipeline did detect any metadata. Will not make annotations for KItem\\n[2024-08-19 19:27:54,455 - dsms_data2rdf.main - INFO]: Checking that dataframe is up to date.\\n[2024-08-19 19:27:54,455 - dsms_data2rdf.main - INFO]: Dataframe upload was successful after 0 retries.\\n[2024-08-19 19:27:54,456 - dsms_data2rdf.main - INFO]: Done!\\n\"\n" + ] + } + ], + "source": [ + "print(job.logs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In case we would like to run the job in the background, we simply add a `wait=False`:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "job = item.kitem_apps.by_title[\"data2rdf\"].run(\n", + " attachment_name=item.attachments[0].name,\n", + " set_token=True,\n", + " set_host_url=True,\n", + " wait=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to monitor the job status and logs asynchronously:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/19/2024, 19:28:06' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Succeeded' estimated_duration=None finished_at='08/19/2024, 19:28:26' started_at='08/19/2024, 19:28:06' message=None progress='1/1'\n", + "\n", + " Current logs:\n", + "\"[2024-08-19 19:28:13,519 - dsms_data2rdf.main - INFO]: Fetch KItem: \\n KItem(\\n\\n\\tname = my tensile test experiment, \\n\\n\\tid = 047c3e0e-0c4c-4815-b041-5251c2ef0882, \\n\\n\\tktype_id = dataset, \\n\\n\\tin_backend = True, \\n\\n\\tslug = mytensiletestexperiment-047c3e0e, \\n\\n\\tannotations = [], \\n\\n\\tattachments = [\\n\\t\\t{\\n\\t\\t\\tname: dummy_data.csv\\n\\t\\t}\\n\\t], \\n\\n\\tlinked_kitems = [], \\n\\n\\taffiliations = [], \\n\\n\\tauthors = [\\n\\t\\t{\\n\\t\\t\\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\\n\\t\\t}\\n\\t], \\n\\n\\tavatar_exists = True, \\n\\n\\tcontacts = [], \\n\\n\\tcreated_at = 2024-08-19 19:27:11.826889, \\n\\n\\tupdated_at = 2024-08-19 19:27:11.826889, \\n\\n\\texternal_links = [], \\n\\n\\tkitem_apps = [\\n\\t\\t{\\n\\t\\t\\tkitem_app_id: 21,\\n\\t\\t\\texecutable: testapp2,\\n\\t\\t\\ttitle: data2rdf,\\n\\t\\t\\tdescription: None,\\n\\t\\t\\ttags: None,\\n\\t\\t\\tadditional_properties: {triggerUponUpload: True, triggerUponUploadFileExtensions: ['.csv']}\\n\\t\\t}\\n\\t], \\n\\n\\tsummary = None, \\n\\n\\tuser_groups = [], \\n\\n\\tcustom_properties = None, \\n\\n\\tdataframe = [\\n\\t\\t{\\n\\t\\t\\tcolumn_id: 0,\\n\\t\\t\\tname: TestTime\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 1,\\n\\t\\t\\tname: StandardForce\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 2,\\n\\t\\t\\tname: AbsoluteCrossheadTravel\\n\\t\\t}\\n\\t], \\n\\n\\trdf_exists = True\\n)\\n[2024-08-19 19:28:13,532 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser arguments: {'metadata_sep': ',', 'metadata_length': '0', 'time_series_sep': ',', 'time_series_header_length': '1', 'drop_na': 'false', 'fillna': ''}\\n[2024-08-19 19:28:13,532 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser: Parser.csv\\n[2024-08-19 19:28:13,532 - dsms_data2rdf.main - INFO]: Run pipeline with the following config: {'base_iri': 'https://bue.materials-data.space/047c3e0e-0c4c-4815-b041-5251c2ef0882', 'data_download_uri': 'https://bue.materials-data.space/api/knowledge/data_api/047c3e0e-0c4c-4815-b041-5251c2ef0882', 'graph_identifier': 'https://bue.materials-data.space/047c3e0e-0c4c-4815-b041-5251c2ef0882', 'separator': '/', 'encoding': 'utf-8'}\\n[2024-08-19 19:28:18,546 - dsms_data2rdf.main - INFO]: Pipeline did detect any metadata. Will not make annotations for KItem\\n[2024-08-19 19:28:18,929 - dsms_data2rdf.main - INFO]: Checking that dataframe is up to date.\\n[2024-08-19 19:28:18,929 - dsms_data2rdf.main - INFO]: Dataframe upload was successful after 0 retries.\\n[2024-08-19 19:28:18,929 - dsms_data2rdf.main - INFO]: Done!\\n\"\n" + ] + } + ], + "source": [ + "while True:\n", + " time.sleep(1)\n", + " print(\"\\n Current status:\")\n", + " print(job.status)\n", + " print(\"\\n Current logs:\")\n", + " print(job.logs)\n", + " if job.status.phase != \"Running\":\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**IMPORTANT**: When job has run asychronously (in the background), we need to manually refresh the KItem afterwards:" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "item.refresh()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Clean up the DSMS from the tutorial" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "del dsms[item]\n", + "del dsms[appspec]\n", + "dsms.commit()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/dsms_sdk/tutorials/testfile.txt b/docs/dsms_sdk/tutorials/testfile.txt new file mode 100755 index 0000000..7976166 --- /dev/null +++ b/docs/dsms_sdk/tutorials/testfile.txt @@ -0,0 +1 @@ +This is a calibration protocol! diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..d500a3a --- /dev/null +++ b/docs/index.md @@ -0,0 +1,74 @@ +# DSMS Documentation + +Welcome to documentation of DSMS! + +Here you will find all the information about DSMS and installation, setup and basic usage of the associated python based DSMS-SDK. + + +````{panels} +:body: text-center. + +--- +**DSMS** + +Introduction to DSMS + +```{link-button} dsms.html +:text: About DSMS +:classes: btn-outline-primary stretched-link + +--- +**DSMS-SDK** + + Overview of the DSMS-SDK + +```{link-button} dsms_sdk.html +:text: Basics of DSMS-SDK +:classes: btn-outline-primary stretched-link + +--- +**Tutorials** + +Get Started with using DSMS-SDK. + +```{link-button} tutorials/1_introduction.html +:text: Jump to the tutorial +:classes: btn-outline-primary stretched-link + + +```` + +Note that these docs are an ongoing effort, so they are likely to change and evolve. +Feel free to report any issues/missing information so we can take a look into it. + +```{toctree} +:hidden: true +:caption: Introduction +:maxdepth: 4 +:glob: + +The Data Space Management System +``` + + +```{toctree} +:hidden: true +:caption: DSMS Python SDK +:maxdepth: 4 +:glob: + +dsms_sdk/dsms_sdk +dsms_sdk/dsms_config_schema +dsms_sdk/dsms_kitem_schema + +``` + +```{toctree} +:hidden: true +:caption: Tutorials +:maxdepth: 4 +:glob: + +dsms_sdk/tutorials/* + +``` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/dsms/__init__.py b/dsms/__init__.py index 5d4f9b7..1ab4066 100644 --- a/dsms/__init__.py +++ b/dsms/__init__.py @@ -1,9 +1,10 @@ """DSMS top module""" +from dsms.apps import AppConfig from dsms.core.configuration import Configuration from dsms.core.context import Context from dsms.core.dsms import DSMS from dsms.knowledge.kitem import KItem from dsms.knowledge.ktype import KType -__all__ = ["DSMS", "Configuration", "Context", "KItem", "KType"] +__all__ = ["DSMS", "Configuration", "Context", "KItem", "KType", "AppConfig"] diff --git a/dsms/apps/__init__.py b/dsms/apps/__init__.py index 6c22c46..fabda28 100644 --- a/dsms/apps/__init__.py +++ b/dsms/apps/__init__.py @@ -1,5 +1,5 @@ """DSMS apps""" -from .apps import App +from .config import AppConfig -__all__ = ["App"] +__all__ = ["AppConfig"] diff --git a/dsms/apps/apps.py b/dsms/apps/apps.py deleted file mode 100644 index b29c607..0000000 --- a/dsms/apps/apps.py +++ /dev/null @@ -1,34 +0,0 @@ -"""DSMS apps models""" - -import urllib.parse - -from pydantic import BaseModel, Field - -from dsms.core.utils import _perform_request - - -class App(BaseModel): - """KItem app list""" - - filename: str = Field( - ..., description="File name of the notebook in the DSMS." - ) - basename: str = Field( - ..., description="Base name of the notebook in the DSMS." - ) - folder: str = Field( - ..., description="Directory of the notebook in the DSMS." - ) - - def get(self, as_html: bool = False) -> str: - """Download the jupyter notebook""" - safe_filename = urllib.parse.quote_plus(self.filename) - response = _perform_request( - f"knowledge/api/apps/{safe_filename}", - "get", - params={"as_html": as_html}, - ) - if not response.ok: - message = f"Something went wrong downloading app `{self.filename}`: {response.text}" - raise RuntimeError(message) - return response.text diff --git a/dsms/apps/config.py b/dsms/apps/config.py new file mode 100644 index 0000000..31d8e3e --- /dev/null +++ b/dsms/apps/config.py @@ -0,0 +1,183 @@ +"""DSMS app models""" +import logging +import urllib.parse +from typing import TYPE_CHECKING, Any, Dict, Union + +import yaml + +from pydantic import ( # isort:skip + BaseModel, + ConfigDict, + Field, + field_validator, + model_validator, +) + +from dsms.apps.utils import ( # isort:skip + _app_spec_exists, + _get_app_specification, +) + + +from dsms.core.logging import handler # isort:skip + + +logger = logging.getLogger(__name__) +logger.addHandler(handler) +logger.propagate = False + +if TYPE_CHECKING: + from dsms import DSMS, Context + + +class AppConfig(BaseModel): + """App config model""" + + name: str = Field(..., description="File name of the app in the DSMS.") + + specification: Union[str, Dict[str, Any]] = Field( + ..., + description="File path for YAML Specification of the app", + ) + + model_config = ConfigDict( + extra="forbid", + validate_assignment=True, + arbitrary_types_allowed=True, + ) + + def __init__(self, **kwargs: "Any") -> None: + """Initialize the KItem""" + from dsms import DSMS + + logger.debug("Initialize KItem with model data: %s", kwargs) + + # set dsms instance if not already done + if not self.dsms: + self.dsms = DSMS() + + # initialize the app config + super().__init__(**kwargs) + + # add app config to buffer + if ( + not self.in_backend + and self.name not in self.context.buffers.created + ): + logger.debug( + """Marking AppConfig with name `%s` as created + and updated during AppConfig initialization.""", + self.name, + ) + self.context.buffers.created.update({self.name: self}) + self.context.buffers.updated.update({self.name: self}) + + logger.debug("AppConfig initialization successful.") + + def __setattr__(self, name, value) -> None: + """Add app to updated-buffer if an attribute is set""" + super().__setattr__(name, value) + logger.debug( + "Setting property with key `%s` on KItem level: %s.", name, value + ) + if self.name not in self.context.buffers.updated: + logger.debug( + "Setting AppConfig with name `%s` as updated during AppConfig.__setattr__", + self.name, + ) + self.context.buffers.updated.update({self.name: self}) + + def __str__(self) -> str: + """Pretty print the app config fields""" + fields = ", ".join( + [ + "{key}={value}".format( # pylint: disable=consider-using-f-string + key=key, + value=( + value + if key != "specification" + else { + "metadata": value.get( # pylint: disable=no-member + "metadata" + ) + } + ), + ) + for key, value in self.__dict__.items() + ] + ) + return f"{self.__class__.__name__}({fields})" + + def __repr__(self) -> str: + """Pretty print the kitem Fields""" + return str(self) + + @field_validator("name") + @classmethod + def validate_name(cls, value: str) -> str: + """Check whether the name of the app contains invalid characters.""" + new_value = urllib.parse.quote_plus(value) + if not new_value == value: + raise ValueError(f"Basename contains invalid characters: {value}") + return value + + @model_validator(mode="after") + @classmethod + def validate_specification(cls, self: "AppConfig") -> str: + """Check specification to be uploaded""" + + if isinstance(self.specification, str): + try: + with open( + self.specification, encoding=self.dsms.config.encoding + ) as file: + content = file.read() + except Exception as error: + raise FileNotFoundError( + f"Invalid file path. File does not exist under path `{self.specification}`." + ) from error + try: + self.specification = yaml.safe_load(content) + except Exception as error: + raise RuntimeError( + f"Invalid yaml specification path: `{error.args[0]}`" + ) from error + self.context.buffers.updated.update({self.name: self}) + elif isinstance(self.specification, dict) and self.in_backend: + spec = _get_app_specification(self.name) + if ( + not yaml.safe_load(spec) == self.specification + and self.name not in self.context.buffers.updated + ): + self.context.buffers.updated.update({self.name: self}) + elif ( + isinstance(self.specification, dict) + and not self.in_backend + and self.name not in self.context.buffers.updated + ): + self.context.buffers.updated.update({self.name: self}) + return self + + @property + def in_backend(self) -> bool: + """Checks whether the app config already exists.""" + return _app_spec_exists(self.name) + + @property + def context(cls) -> "Context": + """Getter for Context""" + from dsms import ( # isort:skip + Context, + ) + + return Context + + @property + def dsms(self) -> "DSMS": + """DSMS context getter""" + return self.context.dsms + + @dsms.setter + def dsms(self, value: "DSMS") -> None: + """DSMS context setter""" + self.context.dsms = value diff --git a/dsms/apps/utils.py b/dsms/apps/utils.py index 8104ee0..6c8e9fb 100644 --- a/dsms/apps/utils.py +++ b/dsms/apps/utils.py @@ -2,18 +2,34 @@ from typing import TYPE_CHECKING -from dsms.apps.apps import App from dsms.core.utils import _perform_request if TYPE_CHECKING: - from typing import List + from typing import Any, Dict, List -def _get_available_apps() -> "List[App]": - """Get available KItem app.""" - response = _perform_request("api/knowledge/apps", "get") +def _get_available_apps_specs() -> "List[Dict[str, Any]]": + """Get available KItem app specs.""" + response = _perform_request("api/knowledge/apps/argo/list", "get") if not response.ok: - message = f"""Something went wrong fetching the available app + message = f"""Something went wrong fetching the available app configs list in the DSMS: {response.text}""" raise RuntimeError(message) - return [App(**app) for app in response.json()] + return response.json() + + +def _app_spec_exists(name: str) -> bool: + """Check whether the specification of the app already exists.""" + response = _perform_request(f"api/knowledge/apps/argo/spec/{name}", "head") + return response.ok + + +def _get_app_specification(appname) -> str: + response = _perform_request( + f"api/knowledge/apps/argo/spec/{appname}", + "get", + ) + if not response.ok: + message = f"Something went wrong downloading app config `{appname}`: {response.text}" + raise RuntimeError(message) + return response.text diff --git a/dsms/core/configuration.py b/dsms/core/configuration.py index 4bd8377..9237366 100644 --- a/dsms/core/configuration.py +++ b/dsms/core/configuration.py @@ -1,14 +1,32 @@ """General config for the DSMS Python SDK""" +import logging import urllib import warnings -from typing import Optional +from enum import Enum +from typing import Callable, Optional, Set, Union import requests -from pydantic import AnyUrl, Field, SecretStr, field_validator +from pydantic import AnyUrl, ConfigDict, Field, SecretStr, field_validator from pydantic_core.core_schema import ValidationInfo from pydantic_settings import BaseSettings, SettingsConfigDict +from .utils import get_callable + +MODULE_REGEX = r"^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*:[a-zA-Z_][a-zA-Z0-9_]*$" +DEFAULT_UNIT_SPARQL = "dsms.knowledge.semantics.units.sparql:UnitSparqlQuery" +DEFAULT_REPO = "knowledge-items" + + +class Loglevel(Enum): + """Enum mapping for default log levels""" + + DEBUG: logging.DEBUG + INFO: logging.INFO + ERROR: logging.ERROR + CRITICAL: logging.CRITICAL + WARNING: logging.WARNING + class Configuration(BaseSettings): """General config for DSMS-SDK""" @@ -17,7 +35,7 @@ class Configuration(BaseSettings): ..., description="Url of the DSMS instance to connect." ) request_timeout: int = Field( - 30, + 120, description="Timeout in seconds until the request to the DSMS is timed out.", ) @@ -43,6 +61,13 @@ class Configuration(BaseSettings): True, description="Check whether the host is a DSMS instance or not." ) + individual_slugs: bool = Field( + True, + description="""When set to `True`, the slugs of the KItems will receive the + first few characters of the KItem-id, when the slug is derived automatically + from the KItem-name.""", + ) + encoding: str = Field( "utf-8", description="General encoding to be used for reading/writing serializations.", @@ -53,11 +78,78 @@ class Configuration(BaseSettings): description="Datetime format used in the DSMS instance.", ) + display_units: bool = Field( + False, + description="""Whether the custom properties or the dataframe columns shall + directly reveal their unit when printed. WARNING: This might lead to performance issues.""", + ) + + autocomplete_units: bool = Field( + True, + description="""When a unit is fetched but does not hold a symbol + next to its URI, it shall be fetched from the respective ontology + (which is general side effect from the `units_sparq_object`.) + WARNING: This might lead to performance issues.""", + ) + kitem_repo: str = Field( - "knowledge", + DEFAULT_REPO, description="Repository of the triplestore for KItems in the DSMS", ) + qudt_units: AnyUrl = Field( + "http://qudt.org/2.1/vocab/unit", + description="URI to QUDT Unit ontology for unit conversion", + ) + + qudt_quantity_kinds: AnyUrl = Field( + "http://qudt.org/vocab/quantitykind/", + description="URI to QUDT quantity kind ontology for unit conversion", + ) + + units_sparql_object: str = Field( + DEFAULT_UNIT_SPARQL, + pattern=MODULE_REGEX, + description="""Class and Module specification in Python for a subclass of + `dsms.knowledge.semantics.units.base:BaseUnitSparqlQuery` in order to retrieve + the units of a DataFrame column/ custom property of a KItem.""", + ) + + hide_properties: Set[Union[str, None]] = Field( + set(), + description="Properties to hide while printing, e.g {'external_links'}", + ) + + loglevel: Optional[Union[Loglevel, str]] = Field( + None, description="Set level of logging messages" + ) + + model_config = ConfigDict(use_enum_values=True) + + @field_validator("loglevel") + def get_loglevel( + cls, val: Optional[Union[Loglevel, str]] + ) -> Optional[Loglevel]: + """Set log level for package""" + if val: + logging.getLogger().setLevel(val) + return val + + @field_validator("units_sparql_object") + def get_unit_sparql_object(cls, val: str) -> "Callable": + """Source the class from the given module""" + return get_callable(val) + + @field_validator("hide_properties") + def validate_hide_properties(cls, val: Set) -> "Callable": + """Source the class from the given module""" + from dsms import KItem + + for key in val: + if key not in KItem.model_fields: + raise KeyError(f"Property `{key}` not in KItem schema") + return val + @field_validator("token") def validate_auth(cls, val, info: ValidationInfo): """Validate the provided authentication/authorization secrets.""" diff --git a/dsms/core/dsms.py b/dsms/core/dsms.py index 61b8ed5..3845509 100644 --- a/dsms/core/dsms.py +++ b/dsms/core/dsms.py @@ -1,8 +1,11 @@ """DSMS connection module""" +import os from typing import TYPE_CHECKING, Any, Dict, List -from dsms.apps.utils import _get_available_apps +from dotenv import load_dotenv + +from dsms.apps.utils import _get_available_apps_specs from dsms.core.configuration import Configuration from dsms.core.context import Context from dsms.core.utils import _ping_dsms @@ -20,23 +23,64 @@ from enum import Enum from typing import Optional - from dsms.apps import App + from dsms.apps import AppConfig from dsms.core.context import Buffers from dsms.knowledge.kitem import KItem from dsms.knowledge.ktype import KType + from dsms.knowledge.search import SearchResult class DSMS: - """General class for connecting and interfacting with DSMS.""" + """ + General class for connecting and interfacing with DSMS. + + This class provides methods to connect to and interact with a DSMS (Data + Space Management System) instance. It abstracts away the complexities of + establishing connections and executing queries. + + Args: + config (Configuration, optional): An optional Configuration object + containing connection details. If not provided, default + configurations will be used. + env (str, optional): An optional string representing the path to the env-file. + This can be used to select environment-specific configurations. Defaults to None. + **kwargs: Configurations can also be set as additional keyword arguments instead of + passing the path to an env-file or the Configuration-object itself. + + """ _context = Context - def __init__(self, config: Configuration = None, **kwargs) -> None: - """Initialize the DSMS object.""" + def __init__( + self, + config: "Optional[Configuration]" = None, + env: "Optional[str]" = None, + **kwargs, + ) -> None: + """Initialize the DSMS object. + Args: + config (Configuration, optional): An optional Configuration object + containing connection details. If not provided, default + configurations will be used. + env (str, optional): An optional string representing the path to the env-file. + This can be used to select environment-specific configurations. Content + of the env-file will be safely loaded using `python-dotenv`. + Hence the env-variables will be pruned once the kernel is closed. + Defaults to None. + **kwargs: Configurations can also be set as additional keyword arguments instead of + passing the path to an env-file or the Configuration-object itself. + """ self._config = None self._context.dsms = self + if env: + if not os.path.exists(env): + raise OSError(f"File `{env}` does not exist") + loaded = load_dotenv(env, verbose=True) + if not loaded: + raise RuntimeError(f"Not able to parse .env file: {env}") + if config is not None and not kwargs: self.config = config elif config is None: @@ -56,20 +100,23 @@ def __getitem__(self, key: str) -> "KItem": """Get KItem from remote DSMS instance.""" return _get_kitem(key) - def __delitem__(self, kitem) -> None: - """Stage an KItem for the deletion. + def __delitem__(self, obj) -> None: + """Stage an KItem, KType or AppConfig for the deletion. WARNING: Changes only will take place after executing the `commit`-method """ - from dsms.knowledge.kitem import ( # isort:skip - KItem, - ) + from dsms import KItem, AppConfig, KType # isort:skip - if not isinstance(kitem, KItem): + if isinstance(obj, KItem): + self.context.buffers.deleted.update({obj.id: obj}) + elif isinstance(obj, AppConfig): + self.context.buffers.deleted.update({obj.name: obj}) + elif isinstance(obj, KType): + raise NotImplementedError("Deletion of KTypes not available yet.") + else: raise TypeError( - f"Object must be of type {KItem}, not {type(kitem)}. " + f"Object must be of type {KItem}, {AppConfig} or {KType}, not {type(obj)}. " ) - kitem.context.buffers.deleted.update({kitem.id: kitem}) def commit(self) -> None: """Commit and empty the buffers of the KItems to the DSMS backend.""" @@ -85,14 +132,14 @@ def search( annotations: "Optional[List[str]]" = [], limit: int = 10, allow_fuzzy: "Optional[bool]" = True, - ) -> "List[KItem]": + ) -> "List[SearchResult]": """Search for KItems in the remote backend.""" return _search(query, ktypes, annotations, limit, allow_fuzzy) @property - def sparql_interface(cls) -> SparqlInterface: + def sparql_interface(self) -> SparqlInterface: """Sparql interface of the DSMS instance.""" - return cls._sparql_interface + return self._sparql_interface @property def ktypes(cls) -> "Enum": @@ -128,7 +175,7 @@ def headers(cls) -> Dict[str, Any]: @property def kitems(cls) -> "List[KItem]": - """KItems instanciated and available in the remote backend. + """KItems instantiated and available in the remote backend. WARNING: This will download _all_ KItems in the backend owned by the current user and may resolve into long response times. The default timeout for requests is defined under the @@ -136,9 +183,14 @@ def kitems(cls) -> "List[KItem]": return _get_kitem_list() @property - def apps(cls) -> "List[App]": - """Return available KItem apps in the DSMS""" - return _get_available_apps() + def app_configs(cls) -> "List[AppConfig]": + """Return available app configs in the DSMS""" + from dsms.apps import AppConfig + + return [ + AppConfig(**app_config) + for app_config in _get_available_apps_specs() + ] @property def buffers(cls) -> "Buffers": diff --git a/dsms/core/logging.py b/dsms/core/logging.py new file mode 100644 index 0000000..b37fddd --- /dev/null +++ b/dsms/core/logging.py @@ -0,0 +1,9 @@ +"""DSMS logging module""" + +import logging +import sys + +LOG_FORMAT = "[%(asctime)s - %(name)s - %(levelname)s]: %(message)s" +handler = logging.StreamHandler(sys.stdout) +formatter = logging.Formatter(LOG_FORMAT) +handler.setFormatter(formatter) diff --git a/dsms/core/utils.py b/dsms/core/utils.py index 33bdd5b..d2382cb 100644 --- a/dsms/core/utils.py +++ b/dsms/core/utils.py @@ -1,12 +1,16 @@ """Core utils of the DSMS core""" import re -from typing import Any +from importlib import import_module +from typing import TYPE_CHECKING from urllib.parse import urljoin from uuid import UUID import requests from requests import Response +if TYPE_CHECKING: + from typing import Any, Callable + def _kitem_id2uri(kitem_id: UUID) -> str: "Convert a kitem id in the DSMS to the full resolvable URI" @@ -27,7 +31,7 @@ def _ping_dsms(): return _perform_request("api/knowledge/docs", "get") -def _perform_request(route: str, method: str, **kwargs: Any) -> Response: +def _perform_request(route: str, method: str, **kwargs: "Any") -> Response: """Perform a general request for a certain route and with a certain method. Kwargs are general arguments which can be passed to the `requests.request`-function. """ @@ -63,3 +67,9 @@ def _name_to_camel(input_string): camel_case_words = [word.title() for word in words] camel_case_string = "".join(camel_case_words) return camel_case_string + + +def get_callable(module: str) -> "Callable": + """Get callable from import-specification""" + module, classname = module.strip().split(":") + return getattr(import_module(module), classname) diff --git a/dsms/knowledge/cli.py b/dsms/knowledge/cli.py new file mode 100644 index 0000000..d52d5fa --- /dev/null +++ b/dsms/knowledge/cli.py @@ -0,0 +1,23 @@ +"""Command line interface for kitem inspection""" + + +import click + +from dsms import DSMS + + +@click.command() +@click.argument("kitem-id") +@click.option( + "-e", "--env", default=".env", help="Env file to load for the dsms session" +) +@click.option("-u", "--units", is_flag=True, help="Display KItem with units") +def lookup_kitem(kitem_id, env, units): + """Simple CLI for looking up KItems from the DSMS""" + dsms = DSMS(env=env, display_units=units) + item = dsms[kitem_id] + click.echo(item) + + +if __name__ == "__main__": + lookup_kitem() diff --git a/dsms/knowledge/kitem.py b/dsms/knowledge/kitem.py index ef5fa1b..7f30f0f 100644 --- a/dsms/knowledge/kitem.py +++ b/dsms/knowledge/kitem.py @@ -1,5 +1,7 @@ """Knowledge Item implementation of the DSMS""" +import json +import logging from datetime import datetime from enum import Enum from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union @@ -15,8 +17,10 @@ Field, ValidationInfo, field_validator, + model_validator, ) +from dsms.core.logging import handler # isort:skip from dsms.knowledge.properties import ( # isort:skip Affiliation, @@ -25,17 +29,17 @@ AnnotationsProperty, App, AppsProperty, + Avatar, Attachment, AttachmentsProperty, Author, AuthorsProperty, ContactInfo, ContactsProperty, - CustomProperties, ExternalLink, ExternalLinksProperty, - KProperty, - HDF5Container, + KItemPropertyList, + DataFrameContainer, Column, LinkedKItem, LinkedKItemsProperty, @@ -48,9 +52,12 @@ from dsms.knowledge.utils import ( # isort:skip _kitem_exists, + _get_kitem, _slug_is_available, _slugify, - _inspect_hdf5, + _inspect_dataframe, + _make_annotation_schema, + _refresh_kitem, ) from dsms.knowledge.sparql_interface.utils import _get_subgraph # isort:skip @@ -59,23 +66,70 @@ from dsms import Context from dsms.core.dsms import DSMS +logger = logging.getLogger(__name__) +logger.addHandler(handler) +logger.propagate = False + class KItem(BaseModel): - """Knowledge Item of the DSMS.""" + """ + Knowledge Item of the DSMS. + + Attributes: + name (str): + Human readable name of the KContext.dsms. + id (Optional[UUID]): + ID of the KItem. Defaults to a new UUID if not provided. + ktype_id (Union[Enum, str]): + Type ID of the KItem. + slug (Optional[str]): + Slug of the KContext.dsms. Minimum length: 4. + annotations (List[Annotation]): + Annotations of the KItem. + attachments (List[Union[Attachment, str]]): + File attachments of the DSMS. + linked_kitems (List[Union[LinkedKItem, "KItem"]]): + KItems linked to the current KItem. + affiliations (List[Affiliation]): + Affiliations related to a KItem. + authors (List[Union[Author, str]]): + Authorship of the KItem. + avatar_exists (Optional[bool]): + Whether the KItem holds an avatar or not. + contacts (List[ContactInfo]): + Contact information related to the KItem. + created_at (Optional[Union[str, datetime]]): + Time and date when the KItem was created. + updated_at (Optional[Union[str, datetime]]): + Time and date when the KItem was updated. + external_links (List[ExternalLink]): + External links related to the KItem. + kitem_apps (List[App]): Apps related to the KItem. + summary (Optional[Union[str, Summary]]): + Human readable summary text of the KItem. + user_groups (List[UserGroup]): + User groups able to access the KItem. + custom_properties (Optional[Any]): + Custom properties associated with the KItem. + dataframe (Optional[Union[List[Column], pd.DataFrame, Dict[str, Union[List, Dict]]]]): + DataFrame interface. + """ # public - name: str = Field( - ..., description="Human readable name of the KContext.dsms" - ) + name: str = Field(..., description="Human readable name of the KItem") id: Optional[UUID] = Field( default_factory=uuid4, description="ID of the KItem", ) ktype_id: Union[Enum, str] = Field(..., description="Type ID of the KItem") + in_backend: bool = Field( + False, + description="Whether the KItem was already created in the backend.", + ) slug: Optional[str] = Field( None, description="Slug of the KContext.dsms", min_length=4 ) - annotations: List[Annotation] = Field( + annotations: List[Union[str, Annotation]] = Field( [], description="Annotations of the KItem" ) attachments: List[Union[Attachment, str]] = Field( @@ -119,21 +173,29 @@ class KItem(BaseModel): description="User groups able to access the KItem.", ) custom_properties: Optional[Any] = Field( - {}, description="Custom properties associated to the KItem" + None, description="Custom properties associated to the KItem" ) ktype: Optional[KType] = Field( None, description="KType of the KItem", exclude=True ) - hdf5: Optional[ + dataframe: Optional[ Union[List[Column], pd.DataFrame, Dict[str, Union[List, Dict]]] - ] = Field(None, description="HDF5 interface.") + ] = Field(None, description="DataFrame interface.") + + rdf_exists: bool = Field( + False, description="Whether the KItem holds an RDF Graph or not." + ) + + avatar: Optional[Union[Avatar, Dict]] = Field( + default_factory=Avatar, description="KItem avatar interface" + ) model_config = ConfigDict( extra="forbid", validate_assignment=True, validate_default=True, - exclude={"ktype"}, + exclude={"ktype", "avatar"}, arbitrary_types_allowed=True, ) @@ -141,32 +203,44 @@ def __init__(self, **kwargs: "Any") -> None: """Initialize the KItem""" from dsms import DSMS + logger.debug("Initialize KItem with model data: %s", kwargs) + # set dsms instance if not already done if not self.dsms: self.dsms = DSMS() - # initalize the kitem + # initialize the kitem super().__init__(**kwargs) # add kitem to buffer - if ( - not _kitem_exists(self) - and self.id not in self.context.buffers.created - ): + if not self.in_backend and self.id not in self.context.buffers.created: + logger.debug( + "Marking KItem with ID `%s` as created and updated during KItem initialization.", + self.id, + ) self.context.buffers.created.update({self.id: self}) self.context.buffers.updated.update({self.id: self}) self._set_kitem_for_properties() + logger.debug("KItem initialization successful.") + def __setattr__(self, name, value) -> None: """Add kitem to updated-buffer if an attribute is set""" super().__setattr__(name, value) + logger.debug( + "Setting property with key `%s` on KItem level: %s.", name, value + ) self._set_kitem_for_properties() if ( self.id not in self.context.buffers.updated and not name.startswith("_") ): + logger.debug( + "Setting KItem with ID `%s` as updated during KItem.__setattr__", + self.id, + ) self.context.buffers.updated.update({self.id: self}) def __str__(self) -> str: @@ -175,7 +249,10 @@ def __str__(self) -> str: [ f"\n\t{key} = {value}" for key, value in self.__dict__.items() - if key not in self.model_config["exclude"] + if ( + key not in self.model_config["exclude"] + and key not in self.dsms.config.hide_properties + ) ] ) return f"{self.__class__.__name__}(\n{fields}\n)" @@ -187,17 +264,43 @@ def __repr__(self) -> str: def __hash__(self) -> int: return hash(str(self)) - @field_validator("affiliations") + @field_validator("affiliations", mode="before") @classmethod - def validate_affiliation( + def validate_affiliations_before( + cls, value: List[Union[str, Affiliation]] + ) -> List[Affiliation]: + """Validate affiliations Field""" + return [ + Affiliation(name=affiliation) + if isinstance(affiliation, str) + else affiliation + for affiliation in value + ] + + @field_validator("affiliations", mode="after") + @classmethod + def validate_affiliation_after( cls, value: List[Affiliation] ) -> AffiliationsProperty: """Validate affiliations Field""" return AffiliationsProperty(value) - @field_validator("annotations") + @field_validator("annotations", mode="before") + @classmethod + def validate_annotations_before( + cls, value: List[Union[str, Annotation]] + ) -> List[Annotation]: + """Validate annotations Field""" + return [ + Annotation(**_make_annotation_schema(annotation)) + if isinstance(annotation, str) + else annotation + for annotation in value + ] + + @field_validator("annotations", mode="after") @classmethod - def validate_annotations( + def validate_annotations_after( cls, value: List[Annotation] ) -> AnnotationsProperty: """Validate annotations Field""" @@ -224,7 +327,7 @@ def validate_attachments_after( """Validate attachments Field""" return AttachmentsProperty(value) - @field_validator("kitem_apps") + @field_validator("kitem_apps", mode="after") @classmethod def validate_apps(cls, value: List[App]) -> AppsProperty: """Validate apps Field""" @@ -241,13 +344,13 @@ def validate_authors(cls, value: List[Author]) -> AuthorsProperty: ] ) - @field_validator("contacts") + @field_validator("contacts", mode="after") @classmethod def validate_contacts(cls, value: List[ContactInfo]) -> ContactsProperty: """Validate contacts Field""" return ContactsProperty(value) - @field_validator("external_links") + @field_validator("external_links", mode="after") @classmethod def validate_external_links( cls, value: List[ExternalLink] @@ -261,24 +364,30 @@ def validate_linked_kitems_list( cls, value: "List[Union[Dict, KItem, Any]]", info: ValidationInfo ) -> List[LinkedKItem]: """Validate each single kitem to be linked""" + src_id = info.data.get("id") linked_kitems = [] for item in value: if isinstance(item, dict): dest_id = item.get("id") if not dest_id: raise ValueError("Linked KItem is missing `id`") + linked_model = _get_kitem(dest_id, as_json=True) elif isinstance(item, KItem): dest_id = item.id + linked_model = item.model_dump() else: try: dest_id = getattr(item, "id") + linked_model = _get_kitem(dest_id, as_json=True) except AttributeError as error: raise AttributeError( f"Linked KItem `{item}` has no attribute `id`." ) from error - linked_kitems.append( - LinkedKItem(id=dest_id, source_id=info.data["id"]) - ) + if str(src_id) == str(dest_id): + raise ValueError( + f"Cannot link KItem with ID `{src_id}` to itself!" + ) + linked_kitems.append(LinkedKItem(**linked_model)) return linked_kitems @field_validator("linked_kitems", mode="after") @@ -290,7 +399,7 @@ def validate_linked_kitems( """Validate the list out of linked KItems""" return LinkedKItemsProperty(value) - @field_validator("user_groups") + @field_validator("user_groups", mode="after") @classmethod def validate_user_groups( cls, value: List[UserGroup] @@ -298,21 +407,6 @@ def validate_user_groups( """Validate user groups Field""" return UserGroupsProperty(value) - @field_validator("custom_properties") - @classmethod - def validate_custom_properties(cls, value: Any) -> CustomProperties: - """Validate custom properties Field""" - if isinstance(value, dict) and "content" in value: - value = CustomProperties(content=value["content"]) - elif isinstance(value, dict): - value = CustomProperties(content=value) - elif not isinstance(value, (CustomProperties, dict, type(None))): - raise TypeError( - f"""`custom_properties` must be of type {CustomProperties} or {dict}, - not {type(value)}.""" - ) - return value - @field_validator("created_at") @classmethod def validate_created(cls, value: str) -> Any: @@ -343,9 +437,8 @@ def validate_ktype(cls, value: KType, info: ValidationInfo) -> KType: """Validate the data attribute of the KItem""" from dsms import Context - ktype_id = info.data.get("ktype_id") - if not value: + ktype_id = info.data.get("ktype_id") if not isinstance(ktype_id, str): value = Context.ktypes.get(ktype_id.value) else: @@ -355,28 +448,44 @@ def validate_ktype(cls, value: KType, info: ValidationInfo) -> KType: raise TypeError( f"KType for `ktype_id={ktype_id}` does not exist." ) + return value + @field_validator("in_backend") + @classmethod + def validate_in_backend(cls, value: bool, info: ValidationInfo) -> bool: + """Checks whether the kitem already exists""" + kitem_id = info.data["id"] + if not value: + value = _kitem_exists(kitem_id) return value @field_validator("slug") @classmethod def validate_slug(cls, value: str, info: ValidationInfo) -> str: """Validate slug""" + from dsms import Context + ktype_id = info.data["ktype_id"] - name = info.data.get("name") kitem_id = info.data.get("id") + kitem_exists = info.data.get("in_backend") + if not isinstance(kitem_exists, bool): + kitem_exists = cls.in_backend + + if not isinstance(ktype_id, str): + ktype = ktype_id.value + else: + ktype = ktype_id + name = info.data.get("name") + if not value: value = _slugify(name) - slugified = _slugify(value) - if len(slugified) < 4: - raise ValueError("Slug length must have a minimum length of 4.") - if value != slugified: - raise ValueError( - f"`{value}` is not a valid slug. A valid variation would be `{slugified}`" - ) - if not _kitem_exists(kitem_id) and not _slug_is_available( - ktype_id.value, value - ): + if len(value) < 4: + raise ValueError( + "Slug length must have a minimum length of 4." + ) + if Context.dsms.config.individual_slugs: + value += f"-{str(kitem_id).split('-', maxsplit=1)[0]}" + if not kitem_exists and not _slug_is_available(ktype, value): raise ValueError(f"Slug for `{value}` is already taken.") return value @@ -388,33 +497,73 @@ def validate_summary(cls, value: Union[str, Summary]) -> Summary: value = Summary(kitem=cls, text=value) return value - @field_validator("hdf5") + @field_validator("avatar", mode="before") @classmethod - def validate_hdf5( + def validate_avatar(cls, value: "Union[Dict, Avatar]") -> Avatar: + """Validate avatar""" + if isinstance(value, dict): + value = Avatar(kitem=cls, **value) + return value + + @field_validator("dataframe") + @classmethod + def validate_dataframe( cls, - value: Union[ - List[Column], pd.DataFrame, Dict[str, Dict[Any, Any]] + value: Optional[ + Union[List[Column], pd.DataFrame, Dict[str, Dict[Any, Any]]] ], # pylint: disable=unused-argument info: ValidationInfo, - ) -> HDF5Container: - """Get HDF5 container if it exists.""" + ) -> DataFrameContainer: + """Get DataFrame container if it exists.""" kitem_id = info.data.get("id") if isinstance(value, (pd.DataFrame, dict)): if isinstance(value, pd.DataFrame): - hdf5 = value.copy(deep=True) - elif isinstance(value, dict): - hdf5 = pd.DataFrame.from_dict(value) + dataframe = value.copy(deep=True) else: - raise TypeError( - f"Data must be of type {dict} or {pd.DataFrame}, not {type(value)}" - ) + dataframe = pd.DataFrame.from_dict(value) else: - columns = _inspect_hdf5(kitem_id) + columns = _inspect_dataframe(kitem_id) if columns: - hdf5 = HDF5Container([Column(**column) for column in columns]) + dataframe = DataFrameContainer( + [Column(**column) for column in columns] + ) else: - hdf5 = None - return hdf5 + dataframe = None + return dataframe + + @model_validator(mode="after") + @classmethod + def validate_custom_properties(cls, self) -> "KItem": + """Validate the custom properties with respect to the KType of the KItem""" + if not isinstance( + self.custom_properties, (BaseModel, dict, type(None)) + ): + raise TypeError( + f"""Custom properties must be one of the following types: + {(BaseModel, dict, type(None))}. Not {type(self.custom_properties)}""" + ) + # validate content with webform model + if self.ktype.webform and isinstance(self.custom_properties, dict): + content = ( + self.custom_properties.get("content") or self.custom_properties + ) + if isinstance(content, str): + try: + content = json.loads(content) + except Exception as error: + raise TypeError( + f"Invalid type: {type(content)}" + ) from error + was_in_buffer = self.id in self.context.buffers.updated + self.custom_properties = self.ktype.webform(**content) + # fix: find a better way to prehebit that properties are + # set in the buffer + if not was_in_buffer: + self.context.buffers.updated.pop(self.id) + # set kitem id for custom properties + if isinstance(self.custom_properties, BaseModel): + self.custom_properties.kitem = self + return self def _set_kitem_for_properties(self) -> None: """Set kitem for CustomProperties and KProperties in order to @@ -422,9 +571,14 @@ def _set_kitem_for_properties(self) -> None: """ for prop in self.__dict__.values(): if ( - isinstance(prop, (KProperty, CustomProperties, Summary)) + isinstance(prop, (KItemPropertyList, Summary, Avatar)) and not prop.kitem ): + logger.debug( + "Setting kitem with ID `%s` for property `%s` on KItem level", + self.id, + type(prop), + ) prop.kitem = self @property @@ -457,5 +611,26 @@ def context(cls) -> "Context": def url(cls) -> str: """URL of the KItem""" return urljoin( - cls.context.dsms.config.host_url, f"{cls.ktype_id}/{cls.slug}" + str(cls.context.dsms.config.host_url), + f"knowledge/{cls._get_ktype_as_str()}/{cls.slug}", + ) + + def is_a(self, to_be_compared: KType) -> bool: + """Check the KType of the KItem""" + return ( + self.ktype_id.value # pylint: disable=no-member + == to_be_compared.value ) + + def refresh(self) -> None: + """Refresh the KItem""" + _refresh_kitem(self) + + def _get_ktype_as_str(self) -> str: + if isinstance(self.ktype_id, str): + ktype = self.ktype_id + elif isinstance(self.ktype_id, Enum): + ktype = self.ktype_id.value # pylint: disable=no-member + else: + raise TypeError(f"Datatype for KType is unknown: {type(ktype)}") + return ktype diff --git a/dsms/knowledge/ktype.py b/dsms/knowledge/ktype.py index 576811e..d695541 100644 --- a/dsms/knowledge/ktype.py +++ b/dsms/knowledge/ktype.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, Field, field_validator, model_serializer -from dsms.knowledge.utils import _parse_model +from dsms.knowledge.utils import _create_custom_properties_model class KType(BaseModel): @@ -15,22 +15,16 @@ class KType(BaseModel): name: Optional[str] = Field( None, description="Human readable name of the KType." ) - form_data: Optional[Any] = Field( - None, description="Form data of the KItem." - ) - data_schema: Optional[Any] = Field( + webform: Optional[Any] = Field(None, description="Form data of the KItem.") + json_schema: Optional[Any] = Field( None, description="OpenAPI schema of the KItem." ) - @field_validator("data_schema") + @field_validator("webform") @classmethod - def validate_data_schema(cls, value) -> Dict[str, Any]: - """Validate the data schema of the ktype""" - if isinstance(value, dict): - value = _parse_model(value) - elif not isinstance(value, BaseModel): - raise TypeError(f"Invalid type for `data_schema`: {type(value)}") - return value + def create_model(cls, value: Optional[Dict[str, Any]]) -> Any: + """Create the datamodel for the ktype""" + return _create_custom_properties_model(value) @model_serializer def serialize(self): diff --git a/dsms/knowledge/properties/__init__.py b/dsms/knowledge/properties/__init__.py index 4d4cd10..13b8547 100644 --- a/dsms/knowledge/properties/__init__.py +++ b/dsms/knowledge/properties/__init__.py @@ -10,10 +10,10 @@ ) from dsms.knowledge.properties.apps import App, AppsProperty from dsms.knowledge.properties.authors import Author, AuthorsProperty -from dsms.knowledge.properties.base import KProperty, KPropertyItem +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList from dsms.knowledge.properties.contacts import ContactInfo, ContactsProperty -from dsms.knowledge.properties.custom_properties import CustomProperties -from dsms.knowledge.properties.hdf5 import Column, HDF5Container +from dsms.knowledge.properties.custom_datatype import NumericalDataType +from dsms.knowledge.properties.dataframe import Column, DataFrameContainer from dsms.knowledge.properties.summary import Summary from dsms.knowledge.properties.user_groups import UserGroup, UserGroupsProperty @@ -33,6 +33,7 @@ ExternalLinksProperty, ) +from dsms.knowledge.properties.avatar import Avatar # isort:skip __all__ = [ "Annotation", @@ -41,7 +42,7 @@ "AttachmentsProperty", "Author", "AuthorsProperty", - "CustomProperties", + "Avatar", "LinkedKItem", "LinkedKItemsProperty", "ContactInfo", @@ -55,8 +56,9 @@ "UserGroupsProperty", "UserGroup", "Summary", - "KProperty", - "KPropertyItem", - "HDF5Container", + "KItemPropertyList", + "KItemProperty", + "DataFrameContainer", "Column", + "NumericalDataType", ] diff --git a/dsms/knowledge/properties/affiliations.py b/dsms/knowledge/properties/affiliations.py index ead24b6..bb0297c 100644 --- a/dsms/knowledge/properties/affiliations.py +++ b/dsms/knowledge/properties/affiliations.py @@ -1,22 +1,23 @@ -"""Affiliation KProperty""" +"""Affiliation property of a KItem""" from typing import TYPE_CHECKING from pydantic import Field -from dsms.knowledge.properties.base import KProperty, KPropertyItem +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList +from dsms.knowledge.properties.utils import _str_to_dict if TYPE_CHECKING: from typing import Callable -class Affiliation(KPropertyItem): +class Affiliation(KItemProperty): """Affiliation of a KItem.""" name: str = Field(..., description="Name of the affiliation") -class AffiliationsProperty(KProperty): +class AffiliationsProperty(KItemPropertyList): """Affiliations property""" # OVERRIDE @@ -24,21 +25,7 @@ class AffiliationsProperty(KProperty): def k_property_item(cls) -> "Callable": return Affiliation - # OVERRIDE - def _add(self, item: Affiliation) -> Affiliation: - """Side effect when an affiliation is added to the KProperty""" - return item - - # OVERRIDE - def _update(self, item: Affiliation) -> Affiliation: - """Side effect when an affiliation is updated at the KProperty""" - return item - - # OVERRIDE - def _get(self, item: Affiliation) -> Affiliation: - """Side effect when getting the affiliation for a specfic kitem""" - return item - - # OVERRIDE - def _delete(self, item: Affiliation) -> None: - """Side effect when deleting the affiliation of a KItem""" + @property + def k_property_helper(cls) -> "Callable": + """Affiliation property helper""" + return _str_to_dict diff --git a/dsms/knowledge/properties/annotations.py b/dsms/knowledge/properties/annotations.py index 27e2974..860c464 100644 --- a/dsms/knowledge/properties/annotations.py +++ b/dsms/knowledge/properties/annotations.py @@ -1,25 +1,29 @@ -"""Annotations KProperty""" +"""Annotation property of a KItem""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from pydantic import Field -from dsms.knowledge.properties.base import KProperty, KPropertyItem +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList +from dsms.knowledge.utils import _make_annotation_schema if TYPE_CHECKING: - from typing import Callable + from typing import Any, Callable, Dict -class Annotation(KPropertyItem): +class Annotation(KItemProperty): """KItem annotation model""" iri: str = Field(..., description="IRI of the annotation") name: str = Field(..., description="Name of the annotation") namespace: str = Field(..., description="Namespace of the annotation") + description: Optional[str] = Field( + None, description="Description of the annotation" + ) -class AnnotationsProperty(KProperty): - """KProperty for annotations""" +class AnnotationsProperty(KItemPropertyList): + """KItemPropertyList for annotations""" # OVERRIDE @property @@ -27,21 +31,12 @@ def k_property_item(cls) -> "Callable": """Annotation data model""" return Annotation - # OVERRIDE - def _add(self, item: Annotation) -> Annotation: - """Side effect when an Annotation is added to the KProperty""" - return item - - # OVERRIDE - def _update(self, item: Annotation) -> Annotation: - """Side effect when an Annotation is updated at the KProperty""" - return item - - # OVERRIDE - def _delete(self, item: Annotation) -> None: - """Side effect when deleting the Annotation of a KItem""" + @property + def k_property_helper(cls) -> None: + """Not defined for Affiliations""" + return _make_annotation_schema - # OVERRIDE - def _get(self, item: Annotation) -> Annotation: - """Side effect when getting the Annotation for a specfic kitem""" - return item + @property + def by_iri(cls) -> "Dict[str, Any]": + """Return dict of annotations per IRI""" + return {annotation.iri for annotation in cls} diff --git a/dsms/knowledge/properties/apps.py b/dsms/knowledge/properties/apps.py index 333676b..6cd69f6 100644 --- a/dsms/knowledge/properties/apps.py +++ b/dsms/knowledge/properties/apps.py @@ -1,10 +1,12 @@ -"""App KProperty""" +"""App property of a KItem""" -from typing import TYPE_CHECKING, Optional +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_serializer -from dsms.knowledge.properties.base import KProperty, KPropertyItem +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList +from dsms.knowledge.utils import _perform_request if TYPE_CHECKING: from typing import Callable @@ -17,13 +19,45 @@ class AdditionalProperties(BaseModel): False, description="Whether the app should be triggered when a file is uploaded", ) - triggerUponUploadFileExtensions: Optional[list[str]] = Field( + triggerUponUploadFileExtensions: Optional[List[str]] = Field( None, description="File extensions for which the upload shall be triggered.", ) + def __str__(self) -> str: + """Pretty print the KItemPropertyList""" + values = ", ".join( + [f"{key}: {value}" for key, value in self.__dict__.items()] + ) + return f"{{{values}}}" -class App(KPropertyItem): + def __repr__(self) -> str: + """Pretty print the Apps""" + return str(self) + + +class JobStatus(BaseModel): + """Status of a job""" + + phase: str = Field(..., description="General job app status") + estimated_duration: Optional[Union[str, datetime]] = Field( + None, description="Estimated duration of the job" + ) + finished_at: Optional[Union[str, datetime]] = Field( + None, description="Datetime when the job was finished" + ) + started_at: Optional[Union[str, datetime]] = Field( + None, description="Datetime when when the job was started" + ) + message: Optional[str] = Field( + None, description="General message of the job" + ) + progress: Optional[str] = Field( + None, description="Relative number of jobs which were finished" + ) + + +class App(KItemProperty): """App of a KItem.""" kitem_app_id: Optional[int] = Field( @@ -32,7 +66,7 @@ class App(KPropertyItem): executable: str = Field( ..., description="Name of the executable related to the app" ) - title: Optional[str] = Field(None, description="Title of the appilcation") + title: str = Field(..., description="Title of the appilcation") description: Optional[str] = Field( None, description="Description of the appilcation" ) @@ -43,13 +77,117 @@ class App(KPropertyItem): None, description="Additional properties related to the appilcation" ) - def run(self, *args, **kwargs) -> None: - """Run application""" - raise NotImplementedError + # OVERRIDE + @model_serializer + def serialize_author(self) -> Dict[str, Any]: + """Serialize author model""" + return { + key: value + for key, value in self.__dict__.items() + if key not in ["id", "kitem_app_id"] + } + + def run( + self, + wait=True, + set_token: bool = False, + set_host_url: bool = False, + **kwargs, + ) -> None: + """Run application. + + Args: + wait (bool, optional): whether the job shall not be run asychronously + (not in the background), but the object should wait until the job finished. + Warning: this may lead to a request timeout for long running jobs! + Job details may not be associated anymore when this occurs. + token (bool, optional): Whether the job also should receive the access + token as paramter.If `True`, the JWT will be set as parameter with + the name ´access_token`. + **kwargs (Any, optional): Additional arguments to be passed to the workflow. + KItem ID is passed automatically + + """ + kwargs["kitem_id"] = str(self.id) + if set_token: + kwargs[ + "access_token" + ] = self.context.dsms.config.token.get_secret_value() + if set_host_url: + kwargs["host_url"] = str(self.context.dsms.config.host_url) + + response = _perform_request( + f"api/knowledge/apps/argo/job/{self.executable}", + "post", + json=kwargs, + params={"wait": wait}, + ) + if not response.ok: + raise RuntimeError( + f"Submission was not successful: {response.text}" + ) + submitted = response.json() + if wait: + self.kitem.refresh() + + return Job(name=submitted.get("name"), executable=self.executable) + + @property + def inputs(self) -> Dict[str, Any]: + """Inputs defined for the app from the webform builder""" + route = f"api/knowledge/apps/argo/{self.executable}/inputs" + response = _perform_request(route, "get") + if not response.ok: + raise RuntimeError( + f"Could not fetch app input schema: {response.text}" + ) + return response.json() + + +class Job(BaseModel): + """Job running an app""" + + name: str = Field(..., description="Name of the job submitted") + + executable: str = Field( + ..., description="Name of the executable of the job" + ) + + @property + def status(self) -> JobStatus: + """Get the status of the currently running job""" + + route = f"api/knowledge/apps/argo/job/{self.name}/status" + + response = _perform_request(route, "get") + if not response.ok: + raise RuntimeError(f"Could not fetch job status: {response.text}") + return JobStatus(**response.json()) + + @property + def artifacts(self) -> Dict[str, Any]: + """Get the atrifcats of a finished job""" + + route = f"api/knowledge/apps/argo/job/{self.name}/artifacts" + response = _perform_request(route, "get") + if not response.ok: + raise RuntimeError( + f"Could not fetch job artifacts: {response.text}" + ) + return response.json() + + @property + def logs(self) -> str: + """Get the logs of a job""" + route = f"api/knowledge/apps/argo/job/{self.name}/logs" + response = _perform_request(route, "get") + if not response.ok: + raise RuntimeError(f"Could not fetch job logs: {response.text}") + return response.text -class AppsProperty(KProperty): - """KProperty for apps""" +class AppsProperty(KItemPropertyList): + """KItemPropertyList for apps""" # OVERRIDE @property @@ -57,21 +195,16 @@ def k_property_item(cls) -> "Callable": """App data model""" return App - # OVERRIDE - def _add(self, item: App) -> App: - """Side effect when an App is added to the KProperty""" - return item - - # OVERRIDE - def _update(self, item: App) -> App: - """Side effect when an App is updated at the KProperty""" - return item + @property + def k_property_helper(cls) -> None: + """Not defined for Apps""" - # OVERRIDE - def _delete(self, item: App) -> None: - """Side effect when deleting the App of a KItem""" + @property + def by_title(cls) -> Dict[str, App]: + """Get apps by title""" + return {app.title: app for app in cls} - # OVERRIDE - def _get(self, item: App) -> App: - """Side effect when getting the App for a specfic kitem""" - return item + @property + def by_exe(cls) -> Dict[str, App]: + """Get apps by executable""" + return {app.executable: app for app in cls} diff --git a/dsms/knowledge/properties/attachments.py b/dsms/knowledge/properties/attachments.py index 425c28e..10df1e7 100644 --- a/dsms/knowledge/properties/attachments.py +++ b/dsms/knowledge/properties/attachments.py @@ -1,28 +1,62 @@ -"""Attachment KProperty""" +"""Attachment property of a KItem""" -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Optional, Union -from pydantic import Field +from pydantic import ConfigDict, Field -from dsms.knowledge.properties.base import KProperty, KPropertyItem +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList +from dsms.knowledge.properties.utils import _str_to_dict from dsms.knowledge.utils import _get_attachment if TYPE_CHECKING: - from typing import Callable + from typing import Any, Callable, Dict, Iterable, List -class Attachment(KPropertyItem): +class Attachment(KItemProperty): """Attachment uploaded by a certain user.""" name: str = Field(..., description="File name of the attachment") - def download(self) -> str: + content: Optional[Union[str, bytes]] = Field( + None, description="Content of the file" + ) + + # OVERRIDE + model_config = ConfigDict( + exclude={"id", "content"}, + populate_by_name=True, + from_attributes=True, + ) + + # OVERRIDE + def __str__(self) -> str: + """Pretty print the Attachment""" + values = ",\n\t\t\t".join( + [ + f"{key}: {value}" + for key, value in self.__dict__.items() + if key not in self.exclude + ] + ) + return f"{{\n\t\t\t{values}\n\t\t}}" + + # OVERRIDE + def __repr__(self) -> str: + """Pretty print the Attachment""" + return str(self) + + def download(self, as_bytes: bool = False) -> "Union[str, bytes]": """Download attachment file""" - return _get_attachment(self.id, self.name) + if not self.content: + content = _get_attachment(self.id, self.name, as_bytes) + else: + content = self.content + return content -class AttachmentsProperty(KProperty): - """KProperty for managing attachments.""" +class AttachmentsProperty(KItemPropertyList): + """KItemPropertyList for managing attachments.""" # OVERRIDE @property @@ -30,20 +64,55 @@ def k_property_item(cls) -> "Callable": return Attachment # OVERRIDE - def _add(self, item: Attachment) -> Attachment: - """Side effect when an Attachment is added to the KProperty""" - return item + @property + def k_property_helper(cls) -> "Callable": + """Helper for constructing attachment property""" + return _str_to_dict - # OVERRIDE - def _update(self, item: Attachment) -> Attachment: - """Side effect when an Attachment is updated at the KProperty""" - return item + def extend(self, iterable: "Iterable") -> None: + """Extend KItemPropertyList with list of KItemProperty""" + from dsms import KItem - # OVERRIDE - def _delete(self, item: Attachment) -> None: - """Side effect when deleting the Attachment of a KItem""" + to_extend = [] + for item in iterable: + if isinstance(item, (list, tuple)): + for subitem in item: + item = self._check_item(subitem) + if not Path(item.name).stem in self.by_name: + to_extend.append(item) + elif isinstance(item, (dict, KItemProperty, KItem)): + item = self._check_item(item) + if not Path(item.name).stem in self.by_name: + to_extend.append(item) + else: + if not Path(item.name).stem in self.by_name: + to_extend.append(item) + if to_extend: + self._mark_as_updated() + super().extend(to_extend) - # OVERRIDE - def _get(self, item: Attachment) -> Attachment: - """Side effect when getting the Attachment for a specfic kitem""" - return item + def append(self, item: "Union[Dict, Any]") -> None: + """Append KItemProperty to KItemPropertyList""" + + item = self._check_item(item) + + if not Path(item.name).stem in self.by_name: + self._mark_as_updated() + super().append(item) + + def insert(self, index: int, item: "Union[Dict, Any]") -> None: + """Insert KItemProperty at KItemPropertyList at certain index""" + + item = self._check_item(item) + if not Path(item.name).stem in self.by_name: + self._mark_as_updated() + super().insert(index, item) + + @property + def by_name(cls) -> "List[str]": + "Return list of names of attachments" + return { + Path(attachment.name).stem + + Path(attachment.name).suffix: attachment + for attachment in cls + } diff --git a/dsms/knowledge/properties/authors.py b/dsms/knowledge/properties/authors.py index e2fd00e..66738f3 100644 --- a/dsms/knowledge/properties/authors.py +++ b/dsms/knowledge/properties/authors.py @@ -1,24 +1,34 @@ -"""Author KProperty""" +"""Author property of a KItem""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict from uuid import UUID -from pydantic import Field +from pydantic import Field, model_serializer -from dsms.knowledge.properties.base import KProperty, KPropertyItem +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList if TYPE_CHECKING: from typing import Callable -class Author(KPropertyItem): +class Author(KItemProperty): """Author of a KItem.""" user_id: UUID = Field(..., description="ID of the DSMS User") + # OVERRIDE + @model_serializer + def serialize_author(self) -> Dict[str, Any]: + """Serialize author model""" + return { + key: str(value) + for key, value in self.__dict__.items() + if key != "id" + } + -class AuthorsProperty(KProperty): - """KProperty for authors""" +class AuthorsProperty(KItemPropertyList): + """KItemPropertyList for authors""" # OVERRIDE @property @@ -27,20 +37,6 @@ def k_property_item(cls) -> "Callable": return Author # OVERRIDE - def _add(self, item: Author) -> Author: - """Side effect when an Author is added to the KProperty""" - return item - - # OVERRIDE - def _update(self, item: Author) -> Author: - """Side effect when an Author is updated at the KProperty""" - return item - - # OVERRIDE - def _get(self, item: Author) -> Author: - """Side effect when getting the Author for a specfic kitem""" - return item - - # OVERRIDE - def _delete(self, item: Author) -> None: - """Side effect when deleting the Author of a KItem""" + @property + def k_property_helper(cls) -> None: + """Not defined for Authors""" diff --git a/dsms/knowledge/properties/avatar.py b/dsms/knowledge/properties/avatar.py new file mode 100644 index 0000000..f2fe1e9 --- /dev/null +++ b/dsms/knowledge/properties/avatar.py @@ -0,0 +1,33 @@ +"""DSMS KItem Avatar""" + +from typing import Optional, Union + +from PIL.Image import Image +from pydantic import ConfigDict, Field + +from dsms.knowledge.properties.base import KItemProperty +from dsms.knowledge.utils import _get_avatar, _make_avatar + + +class Avatar(KItemProperty): + """DSMS KItem Avatar""" + + file: Optional[Union[Image, str]] = Field( + None, + description="The file path to the image when setting a new avatar is set", + ) + include_qr: Optional[bool] = Field( + False, + description="""Whether the new avatar is supposed to include a qr. + This can be combined with an image file.""", + ) + + model_config = ConfigDict(arbitrary_types_allowed=True) + + def download(self) -> "Image": + """Download avatar as PIL Image""" + return _get_avatar(self.kitem) + + def generate(self) -> "Image": + """Generate avatar as PIL Image""" + return _make_avatar(self.kitem, self.file, self.include_qr) diff --git a/dsms/knowledge/properties/base.py b/dsms/knowledge/properties/base.py index 91b7191..f764ae5 100644 --- a/dsms/knowledge/properties/base.py +++ b/dsms/knowledge/properties/base.py @@ -1,4 +1,6 @@ """Basic property for a KItem""" + +import logging from abc import abstractmethod from typing import TYPE_CHECKING, Optional from uuid import UUID @@ -11,9 +13,13 @@ model_serializer, ) +from dsms.core.logging import handler # isort:skip + from dsms.core.utils import _snake_to_camel # isort:skip -from dsms.knowledge.utils import _get_kitem # isort:skip +logger = logging.getLogger(__name__) +logger.addHandler(handler) +logger.propagate = False if TYPE_CHECKING: from typing import Any, Callable, Dict, Iterable, List, Set, Union @@ -21,11 +27,11 @@ from dsms import Context, KItem -class KPropertyItem(BaseModel): - """List item for the KItem-property""" +class KItemProperty(BaseModel): + """Property of a KItem""" id: Optional[UUID] = Field( - None, description="KItem ID related to the KPropertyItem" + None, description="KItem ID related to the KItemProperty" ) model_config = ConfigDict( @@ -38,196 +44,229 @@ class KPropertyItem(BaseModel): _kitem = PrivateAttr(default=None) def __str__(self) -> str: - """Pretty print the KProperty""" - values = ", ".join( + """Pretty print the KItemProperty""" + values = ",\n\t\t\t".join( [ - f"{key}={value}" + f"{key}: {value}" for key, value in self.__dict__.items() if key not in self.exclude ] ) - return f"{self.__class__.__name__}({values})" + return f"{{\n\t\t\t{values}\n\t\t}}" def __repr__(self) -> str: - """Pretty print the KProperty""" + """Pretty print the KItemProperty""" return str(self) - def __setattr__(self, index: int, item: "Any") -> None: + def __setattr__(self, key: str, item: "Any") -> None: """Add KItem to updated buffer.""" - if self._kitem and self.id not in self.context.buffers.updated: - self.context.buffers.updated.update({self.id: self._kitem}) - super().__setattr__(index, item) + logger.debug( + "Setting property with key `%s` on KProperty level: %s.", key, item + ) + if ( + self.kitem + and self.kitem.id not in self.context.buffers.updated + and key not in ["_kitem", "kitem", "id"] + ): + self.context.buffers.updated.update({self.id: self.kitem}) + logger.debug( + "Setting KItem with `%s` as updated KItemProperty.__setattr__" + ) + super().__setattr__(key, item) def __hash__(self) -> int: return hash(str(self)) @property def kitem(cls) -> "KItem": - """KItem related to the KPropertyItem""" - if not cls.id: - raise ValueError("KItem not defined yet for KProperty") - return _get_kitem(cls.id) + """KItem related to the KItemProperty""" + return cls._kitem + + @kitem.setter + def kitem(cls, item: "KItem") -> None: + """Set KItem related to the KItemProperty""" + cls._kitem = item + cls.id = item.id @property def exclude(cls) -> "Optional[Set[str]]": """Fields to be excluded from the JSON-schema""" return cls.model_config.get("exclude") + @property + def context(cls) -> "Context": + """Getter for Context""" + from dsms import ( # isort:skip + Context, + ) + + return Context + @model_serializer def serialize(self): - """Serialize KPropertItem""" + """Serialize KItemProperty""" return { key: value for key, value in self.__dict__.items() if key != "id" } -class KProperty(list): - """Basic class for an KItem-property.""" +class KItemPropertyList(list): + """List of a specific property belonging to a KItem.""" def __init__(self, *args) -> None: self._kitem: "KItem" = None - self.extend(args) + to_extend = self._get_extendables(args) + self.extend(to_extend) @property @abstractmethod def k_property_item(cls) -> "Callable": - """Return the KPropertyItem-class for the KProperty""" - - @abstractmethod - def _add(self, item: KPropertyItem) -> KPropertyItem: - """Side effect when an KPropertyItem is added to the KProperty""" - - @abstractmethod - def _update(self, item: KPropertyItem) -> KPropertyItem: - """Side effect when an KPropertyItem is updated in the KProperty""" - - @abstractmethod - def _get(self, item: KPropertyItem) -> KPropertyItem: - """Side effect when an KPropertyItem is retrieved from the KProperty""" + """Return the KItemProperty-class""" + @property @abstractmethod - def _delete(self, item: KPropertyItem) -> None: - """Side effect when an KPropertyItem is deleted from the KProperty""" + def k_property_helper(cls) -> "Optional[Callable]": + """Optional helper for transforming a given + input into the k property item""" def __str__(self) -> str: - """Pretty print the KProperty""" + """Pretty print the KItemPropertyList""" values = ", \n".join(["\t\t" + repr(value) for value in self]) if values: values = f"\n{values}\n\t" return f"[{values}]" def __repr__(self) -> str: - """Pretty print the KProperty""" + """Pretty print the KItemPropertyList""" return str(self) def __hash__(self) -> int: return hash(str(self)) def __setitem__( - self, index: int, item: "Union[Dict, KPropertyItem]" + self, index: int, item: "Union[Dict, KItemPropertyList]" ) -> None: - """Add or Update KPropertyItem and add it to the updated-buffer.""" + """Add or Update KItemPropertyList and add it to the updated-buffer.""" + logger.debug( + "Setting property with index `%s` on KPropertyList level: %s.", + int, + item, + ) self._mark_as_updated() - item = self._check_k_property_item(item) - if self.kitem: - item.id = self.kitem.id - try: - if self[index] != item: - item = self._update(item) - except IndexError: - item = self._add(item) + item = self._check_item(item) super().__setitem__(index, item) def __delitem__(self, index: int) -> None: - """Delete the KPropertyItem from the KProperty""" + """Delete the KItemPropertyList from the KItemProperty""" - self._mark_as_updated() - item = super().__delitem__(index) - self._delete(item) - - def __getitem__(self, index: int) -> KPropertyItem: - """Get the KPropertyItem from the KProperty""" + logger.debug( + "Deleting property with index `%s` on KPropertyList level", int + ) - item = super().__getitem__(index) - return self._get(item) + self._mark_as_updated() + super().__delitem__(index) def __imul__(self, index: int) -> None: - """Imul the KPropertyItem""" + """Imul the KItemPropertyList""" self._mark_as_updated() super().__imul__(index) - def extend(self, iterable: "Iterable") -> None: - """Extend KProperty with list of KPropertyItem""" + def _get_extendables( + self, iterable: "Iterable" + ) -> "List[KItemPropertyList]": from dsms import KItem to_extend = [] for item in iterable: if isinstance(item, (list, tuple)): for subitem in item: - item = self._check_and_add_item(subitem) + item = self._check_item(subitem) + if not item in self: + to_extend.append(item) + elif isinstance(item, (dict, KItemPropertyList, KItem)): + item = self._check_item(item) + if not item in self: to_extend.append(item) - elif isinstance(item, (dict, KPropertyItem, KItem)): - item = self._check_and_add_item(item) - to_extend.append(item) else: - to_extend.append(item) - self._mark_as_updated() - super().extend(to_extend) + if not item in self: + to_extend.append(item) + return to_extend + + def extend(self, iterable: "Iterable") -> None: + """Extend KItemPropertyList with list of KItemProperty""" + to_extend = self._get_extendables(iterable) + + if to_extend: + logger.debug("Extending KPropertyList with %s.", to_extend) + self._mark_as_updated() + super().extend(to_extend) def append(self, item: "Union[Dict, Any]") -> None: - """Append KPropertyItem to KProperty""" + """Append KItemProperty to KItemPropertyList""" - item = self._check_and_add_item(item) - self._mark_as_updated() - super().append(item) + item = self._check_item(item) + + if not item in self: + logger.debug("Extending KPropertyList with %s.", item) + self._mark_as_updated() + super().append(item) def insert(self, index: int, item: "Union[Dict, Any]") -> None: - """Insert KPropertyItem at KProperty at certain index""" + """Insert KItemProperty at KItemPropertyList at certain index""" - item = self._check_and_add_item(item) - self._mark_as_updated() - super().insert(index, item) + item = self._check_item(item) + if not item in self: + logger.debug("Inserting into KPropertyList: %s.", item) + self._mark_as_updated() + super().insert(index, item) - def pop(self, index=-1) -> KPropertyItem: - """Pop KPropertyItem from KProperty""" + def pop(self, index=-1) -> KItemProperty: + """Pop KItemProperty from KItemPropertyList""" item = super().pop(index) self._mark_as_updated() - self._delete(item) + logger.debug( + "Popping KPropertyList with index `%s`: %s.", index, item + ) return item def remove(self, item: "Union[Dict, Any]") -> None: - """Remove KPropertyItem from KProperty""" + """Remove KItemProperty from KItemPropertyList""" + + logger.debug("Remove from KPropertyList: %s.", item) self._mark_as_updated() - self._delete(item) super().remove(item) - def _check_and_add_item(self, item: "Union[Dict, Any]") -> KPropertyItem: + def _check_item(self, item: "Union[Dict, Any]") -> KItemProperty: item = self._check_k_property_item(item) if self.kitem: - item.id = self.kitem.id - return self._add(item) + item.kitem = self.kitem + return item def _check_k_property_item( self, item: "Union[Dict, Any]" - ) -> KPropertyItem: - """Check the type of the processsed KPropertyItem""" - from dsms import KItem - - if not isinstance(item, (self.k_property_item, dict, KItem, str)): - raise TypeError( - f"""Item `{item}` must be of type {self.k_property_item}, {KItem}, {str} or {dict}, - not `{type(item)}`.""" - ) - if isinstance(item, dict): + ) -> KItemProperty: + """Check the type of the processsed KItemProperty""" + if not isinstance(item, BaseModel): + if self.k_property_helper and not isinstance(item, dict): + item = self.k_property_helper(item) + elif not self.k_property_helper and not isinstance(item, dict): + raise TypeError( + f"""No `k_propertyhelper` defined for {type(self)}. + Hence, item `{item}` must be of type {self.k_property_item}, + {BaseModel} or {dict}, not `{type(item)}`.""" + ) item = self.k_property_item(**item) return item def _mark_as_updated(self) -> None: - """Add KItem of KProperty to updated buffer""" + """Add KItem of KItemPropertyList to updated buffer""" if self._kitem and self._kitem.id not in self.context.buffers.updated: + logger.debug( + "Setting KItem with `%s` as updated on KItemPropertyList level" + ) self.context.buffers.updated.update({self._kitem.id: self._kitem}) @property @@ -249,9 +288,9 @@ def kitem(cls, value: "KItem") -> None: """KItem setter""" cls._kitem = value for item in cls: - item.id = cls.kitem.id + item.kitem = cls.kitem @property def values(cls) -> "List[Dict[str, Any]]": - """Values of the KProperty""" + """Values of the KItemPropertyList""" return list(cls) diff --git a/dsms/knowledge/properties/contacts.py b/dsms/knowledge/properties/contacts.py index 64ac0ba..70353cb 100644 --- a/dsms/knowledge/properties/contacts.py +++ b/dsms/knowledge/properties/contacts.py @@ -1,4 +1,4 @@ -"""DContacts KProperty""" +"""Contacts property of a KItem""" from typing import TYPE_CHECKING, Optional @@ -6,13 +6,13 @@ from pydantic import Field -from dsms.knowledge.properties.base import KProperty, KPropertyItem +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList if TYPE_CHECKING: from typing import Callable -class ContactInfo(KPropertyItem): +class ContactInfo(KItemProperty): """Contact info""" name: str = Field(..., description="Name of the contact person") @@ -22,8 +22,8 @@ class ContactInfo(KPropertyItem): ) -class ContactsProperty(KProperty): - """KProperty for contacts""" +class ContactsProperty(KItemPropertyList): + """KItemPropertyList for contacts""" # OVERRIDE @property @@ -31,20 +31,6 @@ def k_property_item(cls) -> "Callable": return ContactInfo # OVERRIDE - def _add(self, item: ContactInfo) -> ContactInfo: - """Side effect when a ContactInfo is added to the KProperty""" - return item - - # OVERRIDE - def _update(self, item: ContactInfo) -> ContactInfo: - """Side effect when a ContactInfo is updated at the KProperty""" - return item - - # OVERRIDE - def _get(self, item: ContactInfo) -> ContactInfo: - """Side effect when getting the ContactInfo for a specfic kitem""" - return item - - # OVERRIDE - def _delete(self, item: ContactInfo) -> None: - """Side effect when deleting the ContactInfo of a KItem""" + @property + def k_property_helper(cls) -> None: + """Not defined for Contacts""" diff --git a/dsms/knowledge/properties/custom_datatype/__init__.py b/dsms/knowledge/properties/custom_datatype/__init__.py new file mode 100644 index 0000000..f53b7d1 --- /dev/null +++ b/dsms/knowledge/properties/custom_datatype/__init__.py @@ -0,0 +1,5 @@ +"""Custom data type module""" + +from .numerical import NumericalDataType + +__all__ = ["NumericalDataType"] diff --git a/dsms/knowledge/properties/custom_datatype/numerical.py b/dsms/knowledge/properties/custom_datatype/numerical.py new file mode 100644 index 0000000..65c6736 --- /dev/null +++ b/dsms/knowledge/properties/custom_datatype/numerical.py @@ -0,0 +1,97 @@ +"""Module for custom numerical data type""" + +import logging +from typing import TYPE_CHECKING + +from dsms.knowledge.semantics.units.utils import ( + get_conversion_factor, + get_property_unit, +) + +if TYPE_CHECKING: + from typing import Any, Dict, Optional + + from dsms import KItem + + +logger = logging.Logger(__name__) + + +class NumericalDataType(float): + """Custom Base data type for custom properties""" + + def __init__(self, value) -> None: # pylint: disable=unused-argument + self._kitem: "Optional[KItem]" = None + self._name: "Optional[str]" = None + + def __str__(self) -> str: + """Pretty print the numerical datatype""" + if self.kitem.dsms.config.display_units: + try: + string = f"{self.__float__()} {self.get_unit().get('symbol')}" + except Exception as error: + logger.debug( + "Could not fetch unit from `%i`: %i", self.name, error.args + ) + string = str(self.__float__()) + else: + string = str(self.__float__()) + return string + + def __repr__(self) -> str: + return str(self) + + @property + def kitem(cls) -> "Optional[KItem]": + """Context of the current kitem for this property""" + return cls._kitem + + @kitem.setter + def kitem(cls, value: "Optional[KItem]") -> None: + """Setter for current KItem context""" + cls._kitem = value + + @property + def name(cls) -> str: + """Context of the name for this property""" + return cls._name + + @name.setter + def name(cls, value: str) -> None: + """Setter for the name of the property""" + cls._name = value + + def get_unit(self) -> "Dict[str, Any]": + """Get unit for the property""" + return get_property_unit( + self.kitem.id, + self.name, + is_dataframe_column=True, + autocomplete_symbol=self.kitem.dsms.config.autocomplete_units, + ) + + def convert_to( + self, + unit_symbol_or_iri: str, + decimals: "Optional[int]" = None, + use_input_iri: bool = True, + ) -> float: + """ + Convert the data of property to a different unit. + + Args: + unit_symbol_or_iri (str): Symbol or IRI of the unit to convert to. + decimals (Optional[int]): Number of decimals to round the result to. Defaults to None. + use_input_iri (bool): If True, use IRI for unit comparison. Defaults to False. + + Returns: + float: converted value of the property + """ + unit = self.get_unit() + if use_input_iri: + input_str = unit.get("iri") + else: + input_str = unit.get("symbol") + return self * get_conversion_factor( + input_str, unit_symbol_or_iri, decimals=decimals + ) diff --git a/dsms/knowledge/properties/custom_properties.py b/dsms/knowledge/properties/custom_properties.py deleted file mode 100644 index db4a7c9..0000000 --- a/dsms/knowledge/properties/custom_properties.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Custom properties of a KItem""" - - -from typing import TYPE_CHECKING, Any, Optional -from uuid import UUID - -from pydantic import BaseModel, ConfigDict, Field, model_validator - -if TYPE_CHECKING: - from typing import Set, Union - - from dsms import Context, KItem - - -class CustomProperties(BaseModel): - """Model for the custom properties of the KItem""" - - id: Optional[UUID] = Field(None, description="ID of the KItem") - content: Any = Field({}, description="Constent of the custom kitem") - kitem: Optional[Any] = Field( - None, description="KItem related to the CustomProperties", exclude=True - ) - - model_config = ConfigDict( - extra="forbid", exclude={"kitem", "id"}, validate_assignment=True - ) - - def __str__(self) -> str: - """Pretty print the custom properties""" - fields = ", ".join( - [ - f"{key}={value}" - for key, value in self.__dict__.items() - if key not in self.exclude - ] - ) - return f"{self.__class__.__name__}({fields})" - - def __repr__(self) -> str: - """Pretty print the custom properties""" - return str(self) - - def __hash__(self) -> int: - return hash(str(self)) - - def add(self, content: "Union[dict, Any]") -> None: - """Add data for custom properties""" - if isinstance(content, type(None)): - self.content = {} - elif self.data_schema and isinstance(content, dict): - self.content = self.data_schema(content) - elif not self.data_schema and isinstance(content, dict): - self.content = content - elif self.data_schema: - if isinstance(content, self.data_schema): - self.content = content - else: - raise TypeError( - f"""Item `{content}` must be of type {self.data_schema} or {dict}, - not `{type(content)}`.""" - ) - else: - raise TypeError( - f"""Item `{content}` must be of type {self.data_schema} or {dict}, - not `{type(content)}`.""" - ) - self._mark_as_updated() - - def _mark_as_updated(self) -> None: - if self.kitem and self.id not in self.context.buffers.updated: - self.context.buffers.updated.update({self.id: self.kitem}) - - def delete(self) -> None: - """Delete data for custom properties""" - self.content = None - self._mark_as_updated() - - def update(self, content: "Union[dict, Any]") -> None: - """Update data for custom properties""" - self.content = None - self.add(content) - - def get(self) -> Any: - """Get data for custom properties""" - return self.content - - @model_validator(mode="before") - @classmethod - def validate_kitem(cls, data: Any) -> "KItem": - """Validate the custom properties with respect to the KType of the KItem""" - kitem = data.get("kitem") - content = data.get("content") - if kitem: - data["id"] = kitem.id - if kitem.ktype.data_schema: - data["content"] = kitem.ktype.data_schema(**content) - return data - - @property - def id(cls) -> Optional[UUID]: - """Identifier of the KItem related to the CustomProperies""" - if not cls.kitem: - raise ValueError("KItem not defined yet.") - return cls.kitem.id # pylint: disable=E1101 - - @property - def data_schema(cls): - """Data schema related to the ktype of the associated kitem.""" - if not cls.kitem: - raise ValueError("KItem not defined yet.") - return cls.kitem.ktype.data_schema # pylint: disable=E1101 - - @property - def context(cls) -> "Context": - """Getter for Context""" - from dsms import ( # isort:skip - Context, - ) - - return Context - - @property - def exclude(cls) -> "Optional[Set[str]]": - """Fields to be excluded from the JSON-schema""" - return cls.model_config.get("exclude") diff --git a/dsms/knowledge/properties/dataframe.py b/dsms/knowledge/properties/dataframe.py new file mode 100644 index 0000000..b994b4a --- /dev/null +++ b/dsms/knowledge/properties/dataframe.py @@ -0,0 +1,139 @@ +"""DataFrame property of a KItem""" +import logging +from typing import TYPE_CHECKING + +import pandas as pd +from pydantic import Field + +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList +from dsms.knowledge.utils import _get_dataframe_column, _is_number + +from dsms.knowledge.semantics.units import ( # isort:skip + get_conversion_factor, + get_property_unit, +) + +logger = logging.Logger(__name__) + +if TYPE_CHECKING: + from typing import Any, Callable, Dict, List, Optional + + +class Column(KItemProperty): + """ + Column of an DataFrame data frame. + + Attributes: + column_id (int): Column ID in the data frame. + name (str): Name of the column in the data series. + """ + + column_id: int = Field(..., description="Column ID in the data frame") + + name: str = Field( + ..., description="Name of the column in the data series." + ) + + def __repr__(self) -> str: + """Pretty print the numerical datatype""" + if self.kitem and self.kitem.dsms.config.display_units: + try: + unit = f"\tunit={self.get_unit().get('symbol')}\n\t\t" + string = str(self) + string = string[:-1] + unit + string[-1:] + except Exception as error: + logger.debug( + "Could not fetch unit from `%i`: %i", self.name, error.args + ) + string = str(self) + else: + string = str(self) + return string + + def get(self) -> "List[Any]": + """ + Download the data for the column in a time series. + + Returns: + List[Any]: List of data for the column. + """ + return _get_dataframe_column(self.id, self.column_id) + + def get_unit(self) -> "Dict[str, Any]": + """ + Retrieve the unit of the column. + + Returns: + Dict[str, Any]: Dictionary containing unit information. + """ + return get_property_unit( + self.id, + self.name, + is_dataframe_column=True, + autocomplete_symbol=self.kitem.dsms.config.autocomplete_units, + ) + + def convert_to( + self, + unit_symbol_or_iri: str, + decimals: "Optional[int]" = None, + use_input_iri: bool = True, + ) -> "List[Any]": + """ + Convert the data of the column to a different unit. + + Args: + unit_symbol_or_iri (str): Symbol or IRI of the unit to convert to. + decimals (Optional[int]): Number of decimals to round the result to. Defaults to None. + use_input_iri (bool): If True, use IRI for unit comparison. Defaults to False. + + Returns: + List[Any]: List of converted data. + """ + unit = self.get_unit() + if use_input_iri: + input_str = unit.get("iri") + else: + input_str = unit.get("symbol") + factor = get_conversion_factor( + input_str, unit_symbol_or_iri, decimals=decimals + ) + data = self.get() + return [ + number * factor if _is_number(number) else number + for number in data + ] + + +class DataFrameContainer(KItemPropertyList): + """DataFrame container of a data frame related to a KItem""" + + # OVERRIDE + @property + def k_property_item(cls) -> "Callable": + return Column + + # OVERRIDE + @property + def k_property_helper(cls) -> None: + """Not defined for DataFrame""" + + def to_df(self) -> pd.DataFrame: + """Return dataframe as pandas DataFrame""" + data = {column.name: column.get() for column in self} + return pd.DataFrame.from_dict(data) + + def get(self, name: str) -> "Optional[Column]": + """Get a column with a certain name.""" + response = None + for column in self: + if column.name == name: + response = column + return response + + def __getattr__(self, key): + """Return column as attribute.""" + attribute = self.get(key) + if not attribute: + raise AttributeError(f"{self} has no attribute '{key}'") + return attribute diff --git a/dsms/knowledge/properties/external_links.py b/dsms/knowledge/properties/external_links.py index eb74c7e..937d6a7 100644 --- a/dsms/knowledge/properties/external_links.py +++ b/dsms/knowledge/properties/external_links.py @@ -1,24 +1,24 @@ -"""ExternalLink KProperty""" +"""ExternalLink property of a KItem""" from typing import TYPE_CHECKING from pydantic import AnyUrl, Field -from dsms.knowledge.properties.base import KProperty, KPropertyItem +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList if TYPE_CHECKING: from typing import Callable -class ExternalLink(KPropertyItem): +class ExternalLink(KItemProperty): """External link of a KItem.""" label: str = Field(..., description="Label of the external link") url: AnyUrl = Field(..., description="URL of the external link") -class ExternalLinksProperty(KProperty): - """KProperty for external links""" +class ExternalLinksProperty(KItemPropertyList): + """KItemPropertyList for external links""" # OVERRIDE @property @@ -26,20 +26,6 @@ def k_property_item(cls) -> "Callable": return ExternalLink # OVERRIDE - def _add(self, item: ExternalLink) -> ExternalLink: - """Side effect when an ExternalLink is added to the KProperty""" - return item - - # OVERRIDE - def _update(self, item: ExternalLink) -> ExternalLink: - """Side effect when an ExternalLink is updated at the KProperty""" - return item - - # OVERRIDE - def _get(self, item: ExternalLink) -> ExternalLink: - """Side effect when getting the ExternalLink for a specfic kitem""" - return item - - # OVERRIDE - def _delete(self, item: ExternalLink) -> None: - """Side effect when deleting the ExternalLink of a KItem""" + @property + def k_property_helper(cls) -> None: + """Not defined for External links""" diff --git a/dsms/knowledge/properties/hdf5.py b/dsms/knowledge/properties/hdf5.py deleted file mode 100644 index ef634d9..0000000 --- a/dsms/knowledge/properties/hdf5.py +++ /dev/null @@ -1,58 +0,0 @@ -"""HDF5 Properties of a KItem""" -from typing import TYPE_CHECKING - -import pandas as pd -from pydantic import Field - -from dsms.knowledge.properties.base import KProperty, KPropertyItem -from dsms.knowledge.utils import _get_hdf5_column - -if TYPE_CHECKING: - from typing import Any, Callable, List - - -class Column(KPropertyItem): - """Column of an HDF5 data frame""" - - column_id: int = Field(..., description="Column ID in the data frame") - - name: str = Field( - ..., description="Name of the column in the data series." - ) - - def get(self) -> "List[Any]": - """Download the data for the column in a time series""" - return _get_hdf5_column(self.id, self.column_id) - - -class HDF5Container(KProperty): - """HDF5 container of a data frame related to a KItem""" - - # OVERRIDE - @property - def k_property_item(cls) -> "Callable": - return Column - - # OVERRIDE - def _add(self, item: Column) -> Column: - """Side effect when an Column is added to the KProperty""" - return item - - # OVERRIDE - def _update(self, item: Column) -> Column: - """Side effect when an Column is updated at the KProperty""" - return item - - # OVERRIDE - def _delete(self, item: Column) -> None: - """Side effect when deleting the Column of a KItem""" - - # OVERRIDE - def _get(self, item: Column) -> Column: - """Side effect when getting the Column for a specfic kitem""" - return item - - def to_df(self) -> pd.DataFrame: - """Return hdf5 as pandas DataFrame""" - data = {column.name: column.get() for column in self} - return pd.DataFrame.from_dict(data) diff --git a/dsms/knowledge/properties/linked_kitems.py b/dsms/knowledge/properties/linked_kitems.py index 67b5f00..29219e1 100644 --- a/dsms/knowledge/properties/linked_kitems.py +++ b/dsms/knowledge/properties/linked_kitems.py @@ -1,21 +1,75 @@ -"""Linked KItems KProperty""" +"""Linked KItems of a KItem""" - -from typing import TYPE_CHECKING, Optional +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from uuid import UUID -from pydantic import ConfigDict, Field, model_serializer +from pydantic import ( # isort:skip + BaseModel, + ConfigDict, + Field, + PrivateAttr, + model_serializer, + field_validator, +) + +from dsms.knowledge.properties.base import ( # isort:skip + KItemProperty, + KItemPropertyList, +) + +from dsms.knowledge.properties.affiliations import Affiliation # isort:skip +from dsms.knowledge.properties.annotations import Annotation # isort:skip +from dsms.knowledge.properties.attachments import Attachment # isort:skip +from dsms.knowledge.properties.authors import Author # isort:skip +from dsms.knowledge.properties.contacts import ContactInfo # isort:skip +from dsms.knowledge.properties.external_links import ExternalLink # isort:skip +from dsms.knowledge.properties.user_groups import UserGroup # isort:skip +from dsms.knowledge.utils import _get_kitem # isort:skip -from dsms.knowledge.properties.base import KProperty, KPropertyItem -from dsms.knowledge.utils import _get_kitem if TYPE_CHECKING: - from typing import Any, Callable, Dict, Union + from typing import Callable + + from dsms import KItem + +def _linked_kitem_helper(kitem: "KItem"): from dsms import KItem + if not isinstance(kitem, KItem): + raise TypeError(f"{kitem} is not of type {KItem}") + return {"id": kitem.id} + + +class LinkedLinkedKItem(BaseModel): + """Linked KItem of linked KItems""" + + id: str = Field(..., description="ID of a linked KItem of a linked KItem") + + def __str__(self) -> str: + """Pretty print the linked KItems of the linked KItem""" + values = ",\n\t\t\t".join( + [f"{key}: {value}" for key, value in self.__dict__.items()] + ) + return f"{{\n\t\t\t{values}\n\t\t}}" + + def __repr__(self) -> str: + """Pretty print the linked KItems of the linked KItem""" + return str(self) + + +class LinkedKItemSummary(BaseModel): + """Summary of a linked KItem""" + + id: str = Field(..., description="ID of the linked KItem") + + text: str = Field( + ..., description="Text for the summary of the linked KItem" + ) + -class LinkedKItem(KPropertyItem): +class LinkedKItem(KItemProperty): """Data model of a linked KItem""" # OVERRIDE @@ -23,100 +77,161 @@ class LinkedKItem(KPropertyItem): None, description="ID of the KItem to be linked", ) - source_id: Optional[UUID] = Field( - None, description="Source ID of the KItem" + + name: str = Field(..., description="Name of the linked KItem") + + slug: str = Field(..., description="Slug of the linked KItem") + + ktype_id: str = Field(..., description="Ktype ID of the linked KItem") + + summary: Optional[Union[str, LinkedKItemSummary]] = Field( + None, description="Summary of the linked KItem." ) - # OVERRIDE - model_config = ConfigDict(exclude={}) + avatar_exists: bool = Field( + False, description="Wether the linked KItem has an avatar." + ) - # OVERRIDE - def __setattr__(self, index: int, item: "Any") -> None: - """Add KItem to updated buffer.""" - if self._kitem and self.source_id not in self.context.buffers.updated: - self.context.buffers.updated.update({self.source_id: self._kitem}) - super().__setattr__(index, item) + annotations: List[Optional[Annotation]] = Field( + [], description="Annotations of the linked KItem" + ) - # OVERRIDE - @property - def kitem(cls) -> "KItem": - """KItem related to the KPropertyItem""" - if not cls.id: - raise ValueError("KItem not defined yet for KProperty") - return _get_kitem(cls.source_id) + linked_kitems: List[Optional[LinkedLinkedKItem]] = Field( + [], description="Linked KItems of the linked KItem" + ) - # OVERRIDE - @model_serializer - def serialize(self): - """Serialize KPropertItem""" - return {key: str(value) for key, value in self.__dict__.items()} + external_links: List[Optional[ExternalLink]] = Field( + [], description="External links of the linked KItem" + ) + contacts: List[Optional[ContactInfo]] = Field( + [], description="Contact info of the linked KItem" + ) -class LinkedKItemsProperty(KProperty): - """KProperty for linked KItems""" + authors: List[Optional[Author]] = Field( + [], description="Authors of the linked KItem" + ) - # OVERRIDE - @property - def k_property_item(cls) -> "Callable": - return LinkedKItem + linked_affiliations: List[Optional[Affiliation]] = Field( + [], description="Linked affiliations of the linked KItem" + ) - # OVERRIDE - def _add(self, item: LinkedKItem) -> LinkedKItem: - """Side effect when an LinkedKItem is added to the KProperty""" - if self.kitem: - item = self.k_property_item(id=item.id, source_id=self.kitem.id) - else: - item = self.k_property_item(id=item.id) - return item + attachments: List[Union[str, Optional[Attachment]]] = Field( + [], description="Attachment of the linked KItem" + ) - # OVERRIDE - def _update(self, item: LinkedKItem) -> LinkedKItem: - """Side effect when an LinkedKItem is updated at the KProperty""" - return item + user_groups: List[Optional[UserGroup]] = Field( + [], description="User groups of the linked KItem" + ) - # OVERRIDE - def _get(self, item: LinkedKItem) -> LinkedKItem: - """Side effect when getting the LinkedKItem for a specfic kitem""" - return _get_kitem(item.id) + custom_properties: Optional[Any] = Field( + None, description="Custom properies of the linked KItem" + ) + + created_at: Optional[Union[str, datetime]] = Field( + None, description="Time and date when the KItem was created." + ) + updated_at: Optional[Union[str, datetime]] = Field( + None, description="Time and date when the KItem was updated." + ) + + _kitem = PrivateAttr(default=None) # OVERRIDE - def _delete(self, item: LinkedKItem) -> None: - """Side effect when deleting the LinkedKItem of a KItem""" + model_config = ConfigDict(exclude={}, arbitrary_types_allowed=True) # OVERRIDE - def __setitem__( - self, index: int, item: "Union[Dict, KPropertyItem]" - ) -> None: - """Add or Update KPropertyItem and add it to the updated-buffer.""" - - self._mark_as_updated() - item = self._check_k_property_item(item) - if self.kitem: - item.source_id = self.kitem.id - try: - if self[index] != item: - item = self._update(item) - except IndexError: - item = self._add(item) - super().super().__setitem__(index, item) # pylint: disable=no-member + def __str__(self) -> str: + """Pretty print the linked KItem""" + values = "\n\t\t\t".join( + [ + f"{key}: {value}" + for key, value in self.__dict__.items() + if key not in self.exclude + ] + ) + return f"\n\t\t\t{values}\n\t\t" # OVERRIDE - def _check_and_add_item(self, item: "Union[Dict, Any]") -> KPropertyItem: - item = self._check_k_property_item(item) - if self.kitem: - item.source_id = self.kitem.id - return self._add(item) + def __repr__(self) -> str: + """Pretty print the linked KItem""" + return str(self) # OVERRIDE @property def kitem(cls) -> "KItem": - """KItem context of the field""" + """KItem related to the linked KItem""" return cls._kitem # OVERRIDE @kitem.setter def kitem(cls, value: "KItem") -> None: - """KItem setter""" + """Set KItem related to the linked KItem""" cls._kitem = value - for item in cls: - item.source_id = cls.kitem.id + + @field_validator("attachments", mode="before") + @classmethod + def validate_attachments_before( + cls, value: List[Union[str, Attachment]] + ) -> List[Attachment]: + """Validate attachments Field""" + return [ + Attachment(name=attachment) + if isinstance(attachment, str) + else attachment + for attachment in value + ] + + @field_validator("summary", mode="after") + @classmethod + def validate_summary_before( + cls, value: Union[str, LinkedKItemSummary] + ) -> str: + """Validate summary Field""" + if isinstance(value, LinkedKItemSummary): + value = value.text + return value + + # OVERRIDE + @model_serializer + def serialize_author(self) -> Dict[str, Any]: + """Serialize linked kitems model""" + return { + key: ( + str(value) + if key in ["updated_at", "created_at", "id"] + else value + ) + for key, value in self.__dict__.items() + } + + @field_validator("custom_properties") + @classmethod + def validate_custom_properties( + cls, value: "Optional[Dict[str, Any]]" + ) -> "Optional[Dict[str, Any]]": + """Validate the custom properties of the linked KItem""" + if value: + value = value.get("content") or value + return value + + +class LinkedKItemsProperty(KItemPropertyList): + """KItemPropertyList for linked KItems""" + + # OVERRIDE + @property + def k_property_item(cls) -> "Callable": + return LinkedKItem + + # OVERRIDE + @property + def k_property_helper(cls) -> "Callable": + """Linked KItem helper function""" + return _linked_kitem_helper + + def get(self, kitem_id: "Union[str, UUID]") -> "KItem": + """Get the kitem with a certain id which is linked to the source KItem.""" + if not str(kitem_id) in [str(item.id) for item in self]: + raise KeyError(f"A KItem with ID `{kitem_id} is not linked.") + return _get_kitem(kitem_id) diff --git a/dsms/knowledge/properties/summary.py b/dsms/knowledge/properties/summary.py index c8eed0c..ae694a0 100644 --- a/dsms/knowledge/properties/summary.py +++ b/dsms/knowledge/properties/summary.py @@ -1,4 +1,4 @@ -"""Custom properties of a KItem""" +"""Summary of a KItem""" from typing import TYPE_CHECKING, Any, Optional diff --git a/dsms/knowledge/properties/user_groups.py b/dsms/knowledge/properties/user_groups.py index 8e0f2fc..4c5fe7a 100644 --- a/dsms/knowledge/properties/user_groups.py +++ b/dsms/knowledge/properties/user_groups.py @@ -1,45 +1,30 @@ -"""UserGroup KProperty""" +"""UserGroup property of a KItem""" from typing import TYPE_CHECKING from pydantic import Field -from dsms.knowledge.properties.base import KProperty, KPropertyItem +from dsms.knowledge.properties.base import KItemProperty, KItemPropertyList if TYPE_CHECKING: from typing import Callable -class UserGroup(KPropertyItem): +class UserGroup(KItemProperty): """Users groups related to a KItem.""" name: str = Field(..., description="Name of the user group") group_id: str = Field(..., description="ID of the user group") -class UserGroupsProperty(KProperty): - """KProperty for user_groups""" +class UserGroupsProperty(KItemPropertyList): + """KItemPropertyList for user_groups""" @property def k_property_item(cls) -> "Callable": """UserGroup data model""" return UserGroup - # OVERRIDE - def _add(self, item: UserGroup) -> UserGroup: - """Side effect when an UserGroup is added to the KProperty""" - return item - - # OVERRIDE - def _update(self, item: UserGroup) -> UserGroup: - """Side effect when an UserGroup is updated at the KProperty""" - return item - - # OVERRIDE - def _get(self, item: UserGroup) -> UserGroup: - """Side effect when getting the UserGroup for a specfic kitem""" - return item - - # OVERRIDE - def _delete(self, item: UserGroup) -> None: - """Side effect when deleting the UserGroup of a KItem""" + @property + def k_property_helper(cls) -> None: + """Not defined for User groups""" diff --git a/dsms/knowledge/properties/utils.py b/dsms/knowledge/properties/utils.py new file mode 100644 index 0000000..7d68677 --- /dev/null +++ b/dsms/knowledge/properties/utils.py @@ -0,0 +1,10 @@ +"""DSMS KItem Property utils""" + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Dict + + +def _str_to_dict(name: str) -> "Dict[str, str]": + return {"name": name} diff --git a/dsms/knowledge/search.py b/dsms/knowledge/search.py new file mode 100644 index 0000000..42b1e8d --- /dev/null +++ b/dsms/knowledge/search.py @@ -0,0 +1,19 @@ +"""DSMS search model""" + +from typing import TYPE_CHECKING, Union + +from pydantic import BaseModel, Field + +if TYPE_CHECKING: + from dsms import KItem + + +class SearchResult(BaseModel): + """DSMS search result""" + + hit: "KItem" = Field(..., description="KItem returned by the search") + fuzzy: Union[bool, float] = Field( + ..., + description="""Whether the KItem was found through a similarity hit. + If not a bool, a float indicates the distance from search term""", + ) diff --git a/dsms/knowledge/semantics/__init__.py b/dsms/knowledge/semantics/__init__.py new file mode 100644 index 0000000..3c31da6 --- /dev/null +++ b/dsms/knowledge/semantics/__init__.py @@ -0,0 +1 @@ +"""DSMS Semantics Module""" diff --git a/dsms/knowledge/semantics/queries/__init__.py b/dsms/knowledge/semantics/queries/__init__.py new file mode 100644 index 0000000..7d81dc6 --- /dev/null +++ b/dsms/knowledge/semantics/queries/__init__.py @@ -0,0 +1,5 @@ +"""DSMS Semantic Queries""" + +from .base import BaseSparqlQuery + +__all__ = ["BaseSparqlQuery"] diff --git a/dsms/knowledge/semantics/queries/base.py b/dsms/knowledge/semantics/queries/base.py new file mode 100644 index 0000000..37f1b58 --- /dev/null +++ b/dsms/knowledge/semantics/queries/base.py @@ -0,0 +1,76 @@ +"""DSMS Base Abstract Class for Queries""" + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +from rdflib.plugins.sparql.results.jsonresults import JSONResult + +if TYPE_CHECKING: + from typing import Any, Dict, Optional + + from dsms import DSMS + + +class BaseSparqlQuery(ABC): + """Abstract class for DSMS Sparql Query""" + + def __init__(self, **kwargs: "Dict[str, Any]") -> None: + from dsms import Context + + self._kwargs = kwargs + self._results: "Optional[Dict[str, Any]]" = None + self._dsms: "DSMS" = Context.dsms + + self.execute() + + @property + def kwargs(cls) -> "Dict[str, Any]": + """Return kwargs passed during initialization""" + return cls._kwargs + + @property + def dsms(cls) -> "Dict[str, Any]": + """Return dsms context""" + return cls._dsms + + @property + def results(cls) -> "Optional[Dict[str, Any]]": + """Return query results""" + return cls._results + + @property + @abstractmethod + def result_mappings(cls) -> "Dict[str, Any]": + """ + Define a mapping between the output keys and the output datatype. + E.g. {'foo': int, 'bar': str, 'foobar': MyCustomClass} + """ + + @property + @abstractmethod + def query(cls) -> str: + """ + Define sparql query by using the kwargs defined during initialization. + """ + + @abstractmethod + def postprocess_result(cls, row: "Dict[str, Any]") -> "Dict[str, Any]": + """ + Define a function that postprocesses the result of the indivudal row in the + sparql result. This might e.g. be some string operations etc. + """ + + def execute(self) -> None: + """Execute sparql query and bind results.""" + result = self.dsms.sparql_interface.query(self.query) + self._results = [] + for row in JSONResult(result).bindings: + row_converted = {str(key): value for key, value in row.items()} + self._results.append( + self.postprocess_result( + { + name: func(row_converted.get(name)) + for name, func in self.result_mappings.items() + } + ) + ) diff --git a/dsms/knowledge/semantics/units/__init__.py b/dsms/knowledge/semantics/units/__init__.py new file mode 100644 index 0000000..95353a0 --- /dev/null +++ b/dsms/knowledge/semantics/units/__init__.py @@ -0,0 +1,5 @@ +"""Semantic units module of the DSMS""" + +from .utils import get_conversion_factor, get_property_unit + +__all__ = ["get_conversion_factor", "get_property_unit"] diff --git a/dsms/knowledge/semantics/units/base.py b/dsms/knowledge/semantics/units/base.py new file mode 100644 index 0000000..cbc76c2 --- /dev/null +++ b/dsms/knowledge/semantics/units/base.py @@ -0,0 +1,37 @@ +"""DSMS acstract class for units sparql query""" + +from typing import TYPE_CHECKING + +from dsms.knowledge.semantics.queries import BaseSparqlQuery + +if TYPE_CHECKING: + from typing import Any, Dict, Union + from uuid import UUID + + +class BaseUnitSparqlQuery(BaseSparqlQuery): + """ + Abstract class for defining sparql queries fetching + the units of a dataframe column or custom property of + a kitem. + """ + + def __init__( + self, + kitem_id: "Union[str, UUID]", + property_name: str, + is_dataframe_column: bool = False, + autocomplete_symbol: bool = True, + ) -> None: + super().__init__( + kitem_id=kitem_id, + property_name=property_name, + is_dataframe_column=is_dataframe_column, + autocomplete_symbol=autocomplete_symbol, + ) + + # OVERRIDE + @property + def result_mappings(cls) -> "Dict[str, Any]": + """Define mappings for the results of the units sparql queries""" + return {"symbol": str, "iri": str} diff --git a/dsms/knowledge/semantics/units/conversion.py b/dsms/knowledge/semantics/units/conversion.py new file mode 100644 index 0000000..f6f0e98 --- /dev/null +++ b/dsms/knowledge/semantics/units/conversion.py @@ -0,0 +1,140 @@ +"""DSMS Unit Semantics Conversion""" + +from functools import lru_cache +from io import StringIO +from typing import Optional +from urllib.parse import urlparse + +import requests +from rdflib import Graph + + +def _is_valid_url(url: str) -> bool: + try: + result = urlparse(url) + return all([result.scheme, result.netloc]) + except ValueError: + return False + + +def _qudt_sparql(symbol: str) -> str: + return f"""PREFIX qudt: + SELECT DISTINCT ?unit + WHERE {{ + ?unit a qudt:Unit . + {{ + ?unit qudt:symbol "{symbol}" . + }} + UNION + {{ + ?unit qudt:ucumCode "{symbol}"^^qudt:UCUMcs . + }} + }}""" + + +def _qudt_sparql_factor(uri: str) -> str: + return f"""PREFIX qudt: + SELECT DISTINCT ?factor + WHERE {{ + <{uri}> a qudt:Unit ; + qudt:conversionMultiplier ?factor . + }}""" + + +def _sparql_symbol_from_iri(uri: str) -> str: + return f"""PREFIX qudt: + SELECT DISTINCT ?symbol + WHERE {{ + <{uri}> a qudt:Unit ; + qudt:ucumCode ?symbol . + }}""" + + +def _qudt_sparql_quantity(original_uri: str, target_uri: str) -> str: + return f"""PREFIX qudt: + SELECT DISTINCT ?kind + WHERE {{ + ?kind a qudt:QuantityKind ; + qudt:applicableUnit <{original_uri}> , <{target_uri}> . + }}""" + + +@lru_cache +def _units_are_compatible( + original_uri: str, target_uri: str +) -> Optional[bool]: + graph = _get_qudt_graph("qudt_quantity_kinds") + query = _qudt_sparql_quantity(original_uri, target_uri) + quantity = [str(row["kind"]) for row in graph.query(query)] + if len(quantity) == 0: + are_compatiable = False + else: + are_compatiable = True + return are_compatiable + + +@lru_cache +def _check_qudt_mapping(symbol: str) -> Optional[str]: + graph = _get_qudt_graph("qudt_units") + query = _qudt_sparql(symbol) + match = [str(row["unit"]) for row in graph.query(query)] + if len(match) == 0: + raise ValueError( + f"No QUDT Mapping found for unit with symbol `{symbol}`." + ) + if len(match) > 1: + raise ValueError( + f"More than one QUDT Mapping found for unit with symbol `{symbol}`." + ) + return match.pop() + + +@lru_cache +def _get_symbol_from_uri(uri: str) -> str: + graph = _get_qudt_graph("qudt_units") + query = _sparql_symbol_from_iri(uri) + symbol = [str(row["symbol"]) for row in graph.query(query)] + if len(symbol) == 0: + raise ValueError(f"No symbol found for unit with uri `{uri}`.") + if len(symbol) > 1: + raise ValueError( + f"More than one symbol factor for unit with uri `{uri}`." + ) + return symbol.pop() + + +@lru_cache +def _get_factor_from_uri(uri: str) -> int: + graph = _get_qudt_graph("qudt_units") + query = _qudt_sparql_factor(uri) + factor = [float(row["factor"]) for row in graph.query(query)] + if len(factor) == 0: + raise ValueError(f"No conversion factor for unit with uri `{uri}`.") + if len(factor) > 1: + raise ValueError( + f"More than one conversion factor for unit with uri `{uri}`." + ) + return factor.pop() + + +@lru_cache +def _get_qudt_graph(ontology_ref: str) -> Graph: + from dsms import Context + + url = getattr(Context.dsms.config, ontology_ref) + encoding = Context.dsms.config.encoding + graph = Graph() + + response = requests.get(url, timeout=Context.dsms.config.request_timeout) + if response.status_code != 200: + raise RuntimeError( + f"Could not download QUDT ontology. Please check URI: {url}" + ) + response.encoding = encoding + + with StringIO() as tmp: + tmp.write(response.text) + tmp.seek(0) + graph.parse(tmp, encoding=encoding) + + return graph diff --git a/dsms/knowledge/semantics/units/sparql.py b/dsms/knowledge/semantics/units/sparql.py new file mode 100644 index 0000000..a35056a --- /dev/null +++ b/dsms/knowledge/semantics/units/sparql.py @@ -0,0 +1,58 @@ +"""Semantic units for a custom property of a KItem in DSMS""" + +from typing import TYPE_CHECKING +from urllib.parse import urljoin + +from dsms.knowledge.semantics.units.base import BaseUnitSparqlQuery + +from .conversion import _get_symbol_from_uri + +if TYPE_CHECKING: + from typing import Any, Dict + + +class UnitSparqlQuery(BaseUnitSparqlQuery): + """ + Define a sparql query for fetching the units of an HDf5 column of a KItem + """ + + # OVERRIDE + @property + def query(cls) -> str: + """Construct sparql query for getting unit for dataframe column""" + kitem_id = cls.kwargs.get("kitem_id") + property_name = cls.kwargs.get("property_name") + if not kitem_id: + raise ValueError("KItem ID must be defined.") + if not property_name: + raise ValueError("Property name must be defined.") + url = urljoin(str(cls.dsms.config.host_url), str(kitem_id)) + + return f"""prefix csvw: + prefix dcat: + prefix qudt: + prefix xsd: + prefix fileid: <{url}/> + select distinct ?iri + where {{ + bind( + <{url}> as ?g + ) + {{ + graph ?g {{ + fileid:{property_name} qudt:hasUnit ?iri . + }} + }} + }}""" + + # OVERRIDE + def postprocess_result(cls, row: "Dict[str, Any]") -> "Dict[str, Any]": + """ + Define a function that postprocesses the result of the indivudal row in the + sparql result. This might e.g. be some string operations etc. + """ + if row.get("symbol") == "None": + row["symbol"] = None + if cls.kwargs.get("autocomplete_symbol") and not row.get("symbol"): + row["symbol"] = _get_symbol_from_uri(row.get("iri")) + return row diff --git a/dsms/knowledge/semantics/units/utils.py b/dsms/knowledge/semantics/units/utils.py new file mode 100644 index 0000000..6870cca --- /dev/null +++ b/dsms/knowledge/semantics/units/utils.py @@ -0,0 +1,123 @@ +"""Utilities for unit semantics of a KItem""" + +from functools import lru_cache +from typing import TYPE_CHECKING + +from .base import BaseUnitSparqlQuery +from .conversion import ( + _check_qudt_mapping, + _get_factor_from_uri, + _is_valid_url, + _units_are_compatible, +) + +if TYPE_CHECKING: + from typing import Any, Dict, Optional, Union + from uuid import UUID + + +@lru_cache +def get_conversion_factor( + original_unit: str, target_unit: str, decimals: "Optional[int]" = None +) -> float: + """ + Calculate the conversion factor between two compatible units. + + This function calculates the conversion factor between two units, specified + by their URIs or labels. If the provided units are not valid URLs, the + function attempts to check a QUDT (Quantities, Units, Dimensions, and Data + Types in OWL and XML) mapping to retrieve the correct URI. + + Parameters: + original_unit (str): The original unit to convert from, specified by URI or label. + target_unit (str): The target unit to convert to, specified by URI or label. + decimals (Optional[int]): An optional parameter specifying the number of + decimal places to round the conversion factor to. Default is None, + indicating no rounding. + + Returns: + float: The conversion factor from the original unit to the target unit. + + Raises: + ValueError: If the conversion factor cannot be determined due to invalid + unit specifications, missing mapping in QUDT, or if the units are not + compatible for conversion. + + Example: + >>> get_conversion_factor('http://qudt.org/vocab/unit/M', 'http://qudt.org/vocab/unit/IN') + 39.3701 + >>> get_conversion_factor('m', 'in') + 39.3701 + """ + if not _is_valid_url(original_unit): + original_unit = _check_qudt_mapping(original_unit) + if not _is_valid_url(target_unit): + target_unit = _check_qudt_mapping(target_unit) + if not _units_are_compatible(original_unit, target_unit): + raise ValueError( + f"Unit {original_unit} can numerically not be converted into {target_unit}" + ) + factor = _get_factor_from_uri(original_unit) / _get_factor_from_uri( + target_unit + ) + if decimals: + factor = round(factor, decimals) + return factor + + +def get_property_unit( + kitem_id: "Union[str, UUID]", + property_name: str, + is_dataframe_column: bool = False, + autocomplete_symbol: bool = True, +) -> "Dict[str, Any]": + """ + Retrieve the unit associated with a given property of a KIitem. + + Args: + kitem (KItem): The identifier of the KItem. + property_name (str): The name of the property of the KItem for which + the unit is to be retrieved. + is_dataframe_column (bool, optional): Indicates whether the property is an DataFrame + column or a custom property. Defaults to False. + autocomplete_symbol (bool, optional): Whether the symbol of a unit shall be + fetched automatically from the ontology when it is not given next to the + URI. + + Returns: + Dict[str, Any]: A dictionary with the symbol and iri of the unit associated + with the specified property. + + Raises: + ValueError: If unable to retrieve the unit for the property due to any errors or if + the property does not have a unit or has more than one unit associated with it. + """ + from dsms import Context + + units_sparql_object = Context.dsms.config.units_sparql_object + if not issubclass(units_sparql_object, BaseUnitSparqlQuery): + raise TypeError( + f"´{units_sparql_object}´ must be a subclass of `{BaseUnitSparqlQuery}`" + ) + try: + query = units_sparql_object( + kitem_id=kitem_id, + property_name=property_name, + is_dataframe_column=is_dataframe_column, + autocomplete_symbol=autocomplete_symbol, + ) + except Exception as error: + raise ValueError( + f"Something went wrong catching the unit for property `{property_name}`." + ) from error + if len(query.results) == 0: + raise ValueError( + f"""Property `{property_name}` does not own any + unit with respect to the semantics applied.""" + ) + if len(query.results) > 1: + raise ValueError( + f"""Property `{property_name}` owns more than one + unit with respect to the semantics applied.""" + ) + return query.results.pop() diff --git a/dsms/knowledge/sparql_interface/sparql_interface.py b/dsms/knowledge/sparql_interface/sparql_interface.py index f3073bf..118c31d 100644 --- a/dsms/knowledge/sparql_interface/sparql_interface.py +++ b/dsms/knowledge/sparql_interface/sparql_interface.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING +from dsms.core.configuration import DEFAULT_REPO from dsms.knowledge.sparql_interface.subgraph import Subgraph from dsms.knowledge.sparql_interface.utils import ( _add_rdf, @@ -25,7 +26,7 @@ def __init__(self, dsms): self._subgraph = Subgraph(dsms) def query( - self, query: str, repository: str = "knowledge" + self, query: str, repository: str = DEFAULT_REPO ) -> "Dict[str, Any]": """Perform Sparql Query""" return _sparql_query(query, repository) @@ -33,7 +34,7 @@ def query( def update( self, file_or_pathlike: "Union[str, TextIO]", - repository: str = "knowledge", + repository: str = DEFAULT_REPO, ) -> None: """Perform update query from local file""" _sparql_update( @@ -43,7 +44,7 @@ def update( def insert( self, file_or_pathlike: "Union[str, TextIO]", - repository: str = "knowledge", + repository: str = DEFAULT_REPO, ) -> None: """Upload RDF to triplestore from local file""" _add_rdf(file_or_pathlike, self._dsms.config.encoding, repository) diff --git a/dsms/knowledge/sparql_interface/subgraph.py b/dsms/knowledge/sparql_interface/subgraph.py index 9be0f23..66819c1 100644 --- a/dsms/knowledge/sparql_interface/subgraph.py +++ b/dsms/knowledge/sparql_interface/subgraph.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING +from dsms.core.configuration import DEFAULT_REPO from dsms.knowledge.sparql_interface.utils import ( _create_subgraph, _delete_subgraph, @@ -22,22 +23,22 @@ def __init__(self, dsms): """Initalize the Sparql interface""" self._dsms: "DSMS" = dsms - def update(self, graph: "Graph", repository: str = "knowledge") -> None: + def update(self, graph: "Graph", repository: str = DEFAULT_REPO) -> None: """Update a subgraph in the DSMS""" _update_subgraph(graph, self._dsms.config.encoding, repository) - def create(self, graph: "Graph", repository: str = "knowledge") -> None: + def create(self, graph: "Graph", repository: str = DEFAULT_REPO) -> None: """Create a subgraph in the DSMS""" _create_subgraph(graph, self._dsms.config.encoding, repository) - def delete(self, identifier: str, repository: str = "knowledge") -> None: + def delete(self, identifier: str, repository: str = DEFAULT_REPO) -> None: """Delete a subgraph in the DSMS""" _delete_subgraph(identifier, repository) def get( self, identifier: str, - repository: str = "knowledge", + repository: str = DEFAULT_REPO, is_kitem_id: bool = False, ) -> "Graph": """Get a subgraph from the DSMS""" diff --git a/dsms/knowledge/utils.py b/dsms/knowledge/utils.py index 8dbc1f5..277528e 100644 --- a/dsms/knowledge/utils.py +++ b/dsms/knowledge/utils.py @@ -1,49 +1,174 @@ """DSMS knowledge utilities""" +import base64 import io -import json +import logging import re +import warnings from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from uuid import UUID import pandas as pd -from pydantic import BaseModel, create_model +import segno +import yaml +from PIL import Image from requests import Response -from dsms.core.utils import _name_to_camel, _perform_request +from pydantic import ( # isort: skip + BaseModel, + ConfigDict, + Field, + create_model, + model_validator, +) + +from dsms.core.logging import handler # isort:skip + +from dsms.core.utils import _name_to_camel, _perform_request # isort:skip + +from dsms.knowledge.properties.custom_datatype import ( # isort:skip + NumericalDataType, +) + +from dsms.knowledge.search import SearchResult # isort:skip if TYPE_CHECKING: + from dsms.apps import AppConfig from dsms.core.context import Buffers from dsms.knowledge import KItem, KType + from dsms.knowledge.properties import Attachment + +logger = logging.getLogger(__name__) +logger.addHandler(handler) +logger.propagate = False + + +def _is_number(value): + try: + float(value) + return True + except Exception: + return False -def _parse_model(value: Dict[str, Any]) -> BaseModel: +def _create_custom_properties_model( + value: Optional[Dict[str, Any]] +) -> BaseModel: """Convert the dict with the model schema into a pydantic model.""" - title = value.get("title") - properties = value.get("properties") - if not title: - raise KeyError("`data_schema` does not have any `title`.") - if isinstance(properties, type(None)): - raise KeyError("`data_schema` does not have any `properties`.") + from dsms import KItem + fields = {} - for key, props in properties.items(): - dtype = props.get("type") - default = props.get("default") - if dtype == "string": - dtype = str - elif dtype == "integer": - dtype = int - elif dtype == "float": - dtype = float - elif dtype == "bool": - dtype = bool - elif dtype == "object": - dtype = dict - else: - raise TypeError(f"Invalid `type={dtype}`") - fields[key] = (dtype, ... if not default else default) - return create_model(title, **fields) + if isinstance(value, dict): + for item in value.get("sections"): + for form_input in item.get("inputs"): + label = form_input.get("label") + dtype = form_input.get("widget") + default = form_input.get("defaultValue") + slug = _slugify(label) + if dtype in ("Text", "File", "Textarea", "Vocabulary term"): + dtype = Optional[str] + elif dtype in ("Number", "Slider"): + dtype = Optional[NumericalDataType] + elif dtype == "Checkbox": + dtype = Optional[bool] + elif dtype in ("Select", "Radio"): + choices = Enum( + _name_to_camel(label) + "Choices", + { + _name_to_camel(choice["value"]): choice["value"] + for choice in form_input.get("choices") + }, + ) + dtype = Optional[choices] + elif dtype == "Knowledge item": + warnings.warn( + "knowledge item not fully supported for KTypes yet." + ) + dtype = Optional[str] + + fields[slug] = (dtype, default or None) + fields["kitem"] = ( + Optional[KItem], + Field(None, exclude=True), + ) + + config = ConfigDict( + extra="allow", arbitrary_types_allowed=True, exclude={"kitem"} + ) + validators = { + "validate_model": model_validator(mode="before")(_validate_model) + } + model = create_model( + "CustomPropertiesModel", + __config__=config, + __validators__=validators, + **fields, + ) + setattr(model, "__str__", _print_properties) + setattr(model, "__repr__", _print_properties) + setattr(model, "__setattr__", __setattr_property__) + logger.debug("Create custom properties model with fields: %s", fields) + return model + + +def _print_properties(self: Any) -> str: + fields = ", \n".join( + [ + f"\t\t{key}: {value}" + for key, value in self.model_dump().items() + if key not in self.model_config["exclude"] + ] + ) + return f"{{\n{fields}\n\t}}" + + +def __setattr_property__(self, key, value) -> None: + logger.debug( + "Setting property for custom property with key `%s` with value `%s`.", + key, + value, + ) + if _is_number(value): + # convert to convertable numeric object + value = _create_numerical_dtype(key, value, self.kitem) + # mark as updated + if key != "kitem" and self.kitem: + logger.debug( + "Setting related kitem for custom properties with id `%s` as updated", + self.kitem.id, + ) + self.kitem.context.buffers.updated.update({self.kitem.id: self.kitem}) + elif key == "kitem": + # set kitem for convertable numeric datatype + for prop in self.model_dump().values(): + if isinstance(prop, NumericalDataType) and not prop.kitem: + prop.kitem = value + # reassignment of extra fields does not work, seems to be a bug? + # have this workaround: + if key in self.__pydantic_extra__: + self.__pydantic_extra__[key] = value + super(BaseModel, self).__setattr__(key, value) + + +def _create_numerical_dtype( + key: str, value: Union[int, float], kitem: "KItem" +) -> NumericalDataType: + value = NumericalDataType(value) + value.name = key + value.kitem = kitem + return value + + +def _validate_model( + cls, values: Dict[str, Any] # pylint: disable=unused-argument +) -> Dict[str, Any]: + for key, value in values.items(): + if _is_number(value): + values[key] = _create_numerical_dtype( + key, value, values.get("kitem") + ) + return values def _get_remote_ktypes() -> Enum: @@ -58,9 +183,14 @@ def _get_remote_ktypes() -> Enum: raise ConnectionError( f"Something went wrong fetching the remote ktypes: {response.text}" ) + Context.ktypes = {ktype["id"]: KType(**ktype) for ktype in response.json()} - return Enum("KTypes", {_name_to_camel(key): key for key in Context.ktypes}) + ktypes = Enum( + "KTypes", {_name_to_camel(key): key for key in Context.ktypes} + ) + logger.debug("Got the following ktypes from backend: `%s`.", list(ktypes)) + return ktypes def _get_kitem_list() -> "List[KItem]": @@ -91,7 +221,9 @@ def _kitem_exists(kitem: Union[Any, str, UUID]) -> bool: return response.ok -def _get_kitem(uuid: Union[str, UUID]) -> "KItem": +def _get_kitem( + uuid: Union[str, UUID], as_json=False +) -> "Union[KItem, Dict[str, Any]]": """Get the KItem for a instance with a certain ID from remote backend""" from dsms import Context, KItem @@ -106,7 +238,12 @@ def _get_kitem(uuid: Union[str, UUID]) -> "KItem": f"""An error occured fetching the KItem with uuid `{uuid}`: `{response.text}`""" ) - return KItem(**response.json()) + payload = response.json() + if as_json: + response = payload + else: + response = KItem(**payload) + return response def _create_new_kitem(kitem: "KItem") -> None: @@ -117,6 +254,7 @@ def _create_new_kitem(kitem: "KItem") -> None: "slug": kitem.slug, "ktype_id": kitem.ktype.id, } + logger.debug("Create new KItem with payload: %s", payload) response = _perform_request("api/knowledge/kitems", "post", json=payload) if not response.ok: raise ValueError( @@ -124,16 +262,19 @@ def _create_new_kitem(kitem: "KItem") -> None: ) -def _update_kitem(kitem: "KItem") -> Response: +def _update_kitem(new_kitem: "KItem", old_kitem: "Dict[str, Any]") -> Response: """Update a KItem in the remote backend.""" - old_kitem = _get_kitem(kitem.id) - differences = _get_kitems_diffs(old_kitem, kitem) - dumped = kitem.model_dump_json( + differences = _get_kitems_diffs(old_kitem, new_kitem) + payload = new_kitem.model_dump( exclude={ "authors", + "avatar", "annotations", + "custom_properties", "linked_kitems", "updated_at", + "rdf_exists", + "in_backend", "avatar_exists", "user_groups", "ktype_id", @@ -142,32 +283,43 @@ def _update_kitem(kitem: "KItem") -> Response: "kitem_apps", "created_at", "external_links", - "hdf5", + "dataframe", }, exclude_none=True, ) - payload = json.loads(dumped) - payload.update(**differences) payload.update( external_links={ - link.label: str(link.url) for link in kitem.external_links - } + link.label: str(link.url) for link in new_kitem.external_links + }, + **differences, + ) + if new_kitem.custom_properties: + custom_properties = new_kitem.custom_properties.model_dump() + # # a smarted detection whether the custom properties were updated is needed + # old_properties = old_kitem.get("custom_properties") + # if isinstance(old_properties, dict): + # old_custom_properties = old_properties.get("content") + # else: + # old_custom_properties = None + # if custom_properties != old_custom_properties: + # payload.update(custom_properties={"content": custom_properties}) + payload.update(custom_properties={"content": custom_properties}) + logger.debug( + "Update KItem for `%s` with payload: %s", new_kitem.id, payload ) response = _perform_request( - f"api/knowledge/kitems/{kitem.id}", "put", json=payload + f"api/knowledge/kitems/{new_kitem.id}", "put", json=payload ) if not response.ok: raise ValueError( - f"KItem with uuid `{kitem.id}` could not be updated in DSMS: {response.text}`" + f"KItem with uuid `{new_kitem.id}` could not be updated in DSMS: {response.text}`" ) - for key, value in _get_kitem(kitem.id).__dict__.items(): - if key != "attachments": - setattr(kitem, key, value) return response def _delete_kitem(kitem: "KItem") -> None: """Delete a KItem in the remote backend""" + logger.debug("Delete KItem with id: %s", kitem.id) response = _perform_request(f"api/knowledge/kitems/{kitem.id}", "delete") if not response.ok: raise ValueError( @@ -175,23 +327,27 @@ def _delete_kitem(kitem: "KItem") -> None: ) -def _update_attachments(kitem: "KItem") -> None: +def _update_attachments( + new_kitem: "KItem", old_kitem: "Dict[str, Any]" +) -> None: """Update attachments of the KItem.""" - old_kitem = _get_kitem(kitem.id) - differences = _get_attachment_diffs(old_kitem, kitem) + differences = _get_attachment_diffs(old_kitem, new_kitem) + logger.debug( + "Found differences in attachments for kitem with id `%s`: %s", + new_kitem.id, + differences, + ) for upload in differences["add"]: - _upload_attachments(kitem, upload.name) + _upload_attachments(new_kitem, upload) for remove in differences["remove"]: - _delete_attachments(kitem, remove.name) - for key, value in _get_kitem(kitem.id).__dict__.items(): - setattr(kitem, key, value) + _delete_attachments(new_kitem, remove) -def _upload_attachments(kitem: "KItem", attachment: "str") -> None: +def _upload_attachments(kitem: "KItem", attachment: "Attachment") -> None: """Upload the attachments of the KItem""" - path = Path(attachment) + path = Path(attachment.name) - if path.is_file(): + if path.is_file() and not attachment.content: if not path.exists(): raise FileNotFoundError(f"File {path} does not exist.") @@ -206,6 +362,32 @@ def _upload_attachments(kitem: "KItem", attachment: "str") -> None: raise RuntimeError( f"Could not upload attachment `{path}`: {response.text}" ) + elif not path.is_file() and attachment.content: + if isinstance(attachment.content, str): + file = io.StringIO(attachment.content) + elif isinstance(attachment.content, bytes): + file = io.BytesIO(attachment.content) + else: + raise TypeError( + f"""Invalid content type of attachment with name + `{attachment.name}`: {type(attachment.content)}""" + ) + file.name = attachment.name + upload_file = {"dataFile": file} + response = _perform_request( + f"api/knowledge/attachments/{kitem.id}", + "put", + files=upload_file, + ) + if not response.ok: + raise RuntimeError( + f"Could not upload attachment `{path}`: {response.text}" + ) + else: + raise RuntimeError( + f"""Invalid file path, attachment name or attachment content: + name={attachment.name},content={attachment.content}""" + ) def _delete_attachments(kitem: "KItem", file_name: str) -> None: @@ -218,7 +400,9 @@ def _delete_attachments(kitem: "KItem", file_name: str) -> None: ) -def _get_attachment(kitem_id: "KItem", file_name: str) -> str: +def _get_attachment( + kitem_id: "KItem", file_name: str, as_bytes: bool +) -> Union[str, bytes]: """Download attachment from KItem""" url = f"api/knowledge/attachments/{kitem_id}/{file_name}" response = _perform_request(url, "get") @@ -226,78 +410,235 @@ def _get_attachment(kitem_id: "KItem", file_name: str) -> str: raise RuntimeError( f"Download for attachment `{file_name}` was not successful: {response.text}" ) - return response.text + if not as_bytes: + content = response.text + else: + content = response.content + return content + + +def _get_apps_diff( + old_kitem: "Dict[str, Any]", new_kitem: "KItem" +) -> "Dict[str, List[Dict[str, Any]]]": + """Get differences in kitem apps from previous KItem state""" + differences = {} + exclude = {"id", "kitem_app_id"} + old_apps = [ + {key: value for key, value in old.items() if key not in exclude} + for old in old_kitem.get("kitem_apps") + ] + new_apps = [new.model_dump() for new in new_kitem.kitem_apps] + differences["kitem_apps_to_update"] = [ + attr for attr in new_apps if attr not in old_apps + ] + differences["kitem_apps_to_remove"] = [ + attr for attr in old_apps if attr not in new_apps + ] + logger.debug("Found differences in KItem apps: %s", differences) + return differences -def _get_attachment_diffs(kitem_old: "KItem", kitem_new: "KItem"): +def _get_linked_diffs( + old_kitem: "Dict[str, Any]", new_kitem: "KItem" +) -> "Dict[str, List[Dict[str, UUID]]]": + """Get differences in linked kitem from previous KItem state""" + differences = {} + old_linked = [old.get("id") for old in old_kitem.get("linked_kitems")] + new_linked = [str(new_kitem.id) for new_kitem in new_kitem.linked_kitems] + differences["kitems_to_link"] = [ + {"id": attr} for attr in new_linked if attr not in old_linked + ] + differences["kitems_to_unlink"] = [ + {"id": attr} for attr in old_linked if attr not in new_linked + ] + logger.debug("Found differences in linked KItems: %s", differences) + return differences + + +def _get_attachment_diffs(kitem_old: "Dict[str, Any]", kitem_new: "KItem"): """Check which attachments should be removed and which should be added.""" + old_attachments = [ + attachment.get("name") + for attachment in kitem_old.get("attachments") + if attachment.get("name") + ] return { - "remove": set(kitem_old.attachments) - set(kitem_new.attachments), - "add": set(kitem_new.attachments) - set(kitem_old.attachments), + "remove": [ + attachment + for attachment in old_attachments + if attachment not in kitem_new.attachments.by_name + ], + "add": [ + attachment + for name, attachment in kitem_new.attachments.by_name.items() + if name not in old_attachments or attachment.content + ], } -def _get_kitems_diffs(kitem_old: "KItem", kitem_new: "KItem"): +def _get_kitems_diffs(kitem_old: "Dict[str, Any]", kitem_new: "KItem"): """Get the differences in the attributes between two kitems""" differences = {} attributes = [ - ("linked_kitems", ("kitems", "link", "unlink")), ("annotations", ("annotations", "link", "unlink")), - ("kitem_apps", ("kitem_apps", "update", "remove")), ("user_groups", ("user_groups", "add", "remove")), ] + to_compare = kitem_new.model_dump(include={"annotations", "user_groups"}) for name, terms in attributes: to_add_name = terms[0] + "_to_" + terms[1] to_remove_name = terms[0] + "_to_" + terms[2] - old_attr = getattr(kitem_old, name) - new_attr = getattr(kitem_new, name) + old_attr = kitem_old.get(name) + new_attr = to_compare.get(name) differences[to_add_name] = [ - json.loads(attr.model_dump_json()) - for attr in set(new_attr) - set(old_attr) + attr for attr in new_attr if attr not in old_attr ] differences[to_remove_name] = [ - json.loads(attr.model_dump_json()) - for attr in set(old_attr) - set(new_attr) + attr for attr in old_attr if attr not in new_attr ] + logger.debug( + "Found differences between new and old KItem: %s", differences + ) + # linked kitems need special treatment since the linked target + # kitems also might differ in their new properties in some cases. + linked_kitems = _get_linked_diffs(kitem_old, kitem_new) + # same holds for kitem apps + kitem_apps = _get_apps_diff(kitem_old, kitem_new) + # merge with previously found differences + differences.update(**linked_kitems, **kitem_apps) return differences def _commit(buffers: "Buffers") -> None: """Commit the buffers for the created, updated and deleted buffers""" + logger.debug("Committing KItems in buffers. Current buffers:") + logger.debug("Current Created-buffer: %s", buffers.created) + logger.debug("Current Updated-buffer: %s", buffers.updated) + logger.debug("Current Deleted-buffer: %s", buffers.deleted) _commit_created(buffers.created) _commit_updated(buffers.updated) _commit_deleted(buffers.deleted) + logger.debug("Committing successful, clearing buffers.") -def _commit_created(buffer: "Dict[str, KItem]") -> dict: +def _commit_created( + buffer: "Dict[str, Union[KItem, KType, AppConfig]]", +) -> dict: """Commit the buffer for the `created` buffers""" - for kitem in buffer.values(): - exists = _kitem_exists(kitem) - if not exists: - _create_new_kitem(kitem) + from dsms import AppConfig, KItem, KType + + for obj in buffer.values(): + if isinstance(obj, KItem): + _create_new_kitem(obj) + elif isinstance(obj, AppConfig): + _create_or_update_app_spec(obj) + elif isinstance(obj, KType): + raise NotImplementedError( + "Committing of KTypes not implemented yet." + ) + else: + raise TypeError( + f"Object `{obj}` of type {type(obj)} cannot be committed." + ) -def _commit_updated(buffer: "Dict[str, KItem]") -> None: +def _commit_updated( + buffer: "Dict[str, Union[KItem, AppConfig, KType]]", +) -> None: """Commit the buffer for the `updated` buffers""" - for kitem in buffer.values(): - if _kitem_exists(kitem): - if isinstance(kitem.hdf5, pd.DataFrame): - _update_hdf5(kitem.id, kitem.hdf5) - elif isinstance(kitem.hdf5, type(None)) and _inspect_hdf5( - kitem.id - ): - _delete_hdf5(kitem.id) - _update_kitem(kitem) - _update_attachments(kitem) - - -def _commit_deleted(buffer: "Dict[str, KItem]") -> None: + from dsms import AppConfig, KItem, KType + + for obj in buffer.values(): + if isinstance(obj, KItem): + _commit_updated_kitem(obj) + elif isinstance(obj, AppConfig): + _create_or_update_app_spec(obj, overwrite=True) + elif isinstance(obj, KType): + raise NotImplementedError( + "Committing of KTypes not implemented yet." + ) + else: + raise TypeError( + f"Object `{obj}` of type {type(obj)} cannot be committed." + ) + + +def _commit_updated_kitem(new_kitem: "KItem") -> None: + """Commit the updated KItems""" + old_kitem = _get_kitem(new_kitem.id, as_json=True) + logger.debug( + "Fetched data from old KItem with id `%s`: %s", + new_kitem.id, + old_kitem, + ) + if old_kitem: + if isinstance(new_kitem.dataframe, pd.DataFrame): + logger.debug( + "New KItem data has `pd.DataFrame`. Will push as dataframe." + ) + _update_dataframe(new_kitem.id, new_kitem.dataframe) + new_kitem.dataframe = _inspect_dataframe(new_kitem.id) + elif isinstance( + new_kitem.dataframe, type(None) + ) and _inspect_dataframe(new_kitem.id): + _delete_dataframe(new_kitem.id) + _update_kitem(new_kitem, old_kitem) + _update_attachments(new_kitem, old_kitem) + if new_kitem.avatar.file or new_kitem.avatar.include_qr: + _commit_avatar(new_kitem) + new_kitem.in_backend = True + logger.debug( + "Fetching updated KItem from remote backend: %s", new_kitem.id + ) + new_kitem.refresh() + + +def _commit_deleted( + buffer: "Dict[str, Union[KItem, KType, AppConfig]]", +) -> None: """Commit the buffer for the `deleted` buffers""" - for kitem in buffer.values(): - if _kitem_exists(kitem): - _delete_hdf5(kitem.id) - _delete_kitem(kitem) + from dsms import AppConfig, KItem, KType + + for obj in buffer.values(): + if isinstance(obj, KItem): + _delete_dataframe(obj.id) + _delete_kitem(obj) + elif isinstance(obj, AppConfig): + _delete_app_spec(obj.name) + elif isinstance(obj, KType): + raise NotImplementedError( + "Deletion of KTypes not implemented yet." + ) + else: + raise TypeError( + f"Object `{obj}` of type {type(obj)} cannot be committed or deleted." + ) + + +def _refresh_kitem(kitem: "KItem") -> None: + """Refresh the KItem""" + for key, value in _get_kitem(kitem.id, as_json=True).items(): + logger.debug( + "Set updated property `%s` for KItem with id `%s` after commiting: %s", + key, + kitem.id, + value, + ) + setattr(kitem, key, value) + kitem.dataframe = _inspect_dataframe(kitem.id) + + +def _split_iri(iri: str) -> List[str]: + if "#" in iri: + namspace, name = iri.rsplit("#", 1) + else: + namspace, name = iri.rsplit("/", 1) + return namspace, name + + +def _make_annotation_schema(iri: str) -> Dict[str, Any]: + namespace, name = _split_iri(iri) + return {"namespace": namespace, "name": name, "iri": iri} def _search( @@ -306,14 +647,14 @@ def _search( annotations: "Optional[List[str]]" = [], limit: "Optional[int]" = 10, allow_fuzzy: "Optional[bool]" = True, -) -> "List[KItem]": +) -> "List[SearchResult]": """Search for KItems in the remote backend""" - from dsms import KItem # isort:skip + from dsms import KItem payload = { "search_term": query or "", "ktypes": [ktype.value for ktype in ktypes], - "kitem_annotations": annotations, + "annotations": [_make_annotation_schema(iri) for iri in annotations], "limit": limit, } response = _perform_request( @@ -332,13 +673,16 @@ def _search( raise RuntimeError( f"""Something went wrong while searching for KItems: {response.text}""" ) from excep - return [KItem(**item) for item in dumped] + return [ + SearchResult(hit=KItem(**item.get("hit")), fuzzy=item.get("fuzzy")) + for item in dumped + ] -def _slugify(input_string): +def _slugify(input_string: str, replacement: str = ""): """Turn any arbitrary string into a slug.""" slug = re.sub( - r"[^\w\s\-_]", "", input_string + r"[^\w\s\-_]", replacement, input_string ) # Remove all non-word characters (everything except numbers and letters) slug = re.sub(r"\s+", "", slug) # Replace all runs of whitespace slug = slug.lower() # Convert the string to lowercase. @@ -350,11 +694,14 @@ def _slug_is_available(ktype_id: Union[str, UUID], value: str) -> bool: response = _perform_request( f"api/knowledge/kitems/{ktype_id}/{value}", "head" ) + if response.status_code == 401: + raise RuntimeError("The access token has expired") return response.status_code == 404 -def _get_hdf5_column(kitem_id: str, column_id: int) -> List[Any]: - """Download the column of a hdf5 container of a certain kitem""" +def _get_dataframe_column(kitem_id: str, column_id: int) -> List[Any]: + """Download the column of a dataframe container of a certain kitem""" + response = _perform_request( f"api/knowledge/data_api/{kitem_id}/column-{column_id}", "get" ) @@ -365,32 +712,134 @@ def _get_hdf5_column(kitem_id: str, column_id: int) -> List[Any]: return response.json().get("array") -def _inspect_hdf5(kitem_id: str) -> Optional[List[Dict[str, Any]]]: - """Get column info for the hdf5 container of a certain kitem""" +def _inspect_dataframe(kitem_id: str) -> Optional[List[Dict[str, Any]]]: + """Get column info for the dataframe container of a certain kitem""" response = _perform_request(f"api/knowledge/data_api/{kitem_id}", "get") if not response.ok and response.status_code == 404: - hdf5 = None + dataframe = None elif not response.ok and response.status_code != 404: message = f"""Something went wrong fetching intospection for kitem `{kitem_id}`: {response.text}""" raise ValueError(message) else: - hdf5 = response.json() - return hdf5 + dataframe = response.json() + return dataframe + + +def _update_dataframe(kitem_id: str, data: pd.DataFrame): + if data.empty: + _delete_dataframe(kitem_id) + else: + buffer = io.BytesIO() + data.to_json(buffer, indent=2) + buffer.seek(0) + response = _perform_request( + f"api/knowledge/data_api/{kitem_id}", "put", files={"data": buffer} + ) + if not response.ok: + raise RuntimeError( + f"Could not put dataframe into kitem with id `{kitem_id}`: {response.text}" + ) + + +def _delete_dataframe(kitem_id: str) -> Response: + logger.debug("Delete DataFrame for kitem with id `%s`.", kitem_id) + return _perform_request(f"api/knowledge/data_api/{kitem_id}", "delete") -def _update_hdf5(kitem_id: str, data: pd.DataFrame): +def _commit_avatar(kitem) -> None: + if kitem.avatar_exists: + response = _perform_request( + f"api/knowledge/avatar/{kitem.id}", "delete" + ) + if not response.ok: + message = ( + f"Something went wrong deleting the avatar: {response.text}" + ) + raise RuntimeError(message) + avatar = kitem.avatar.generate() buffer = io.BytesIO() - data.to_json(buffer, indent=2) + avatar.save(buffer, "JPEG", quality=100) buffer.seek(0) + encoded_image = base64.b64encode(buffer.getvalue()) + encoded_image_str = "data:image/jpeg;base64," + encoded_image.decode( + "utf-8" + ) response = _perform_request( - f"api/knowledge/data_api/{kitem_id}", "put", files={"data": buffer} + f"api/knowledge/avatar/{kitem.id}", + "put", + json={ + "croppedImage": encoded_image_str, + "originalImage": encoded_image_str, + "filename": kitem.name + ".jpeg", + }, ) if not response.ok: raise RuntimeError( - f"Could not put dataframe into kitem with id `{kitem_id}`: {response.text}" + f"Something went wrong while updating the avatar: {response.text}" ) -def _delete_hdf5(kitem_id: str) -> Response: - return _perform_request(f"api/knowledge/data_api/{kitem_id}", "delete") +def _make_avatar( + kitem: "KItem", image: Optional[Union[str, Image.Image]], make_qr: bool +) -> Image.Image: + avatar = None + if make_qr: + # this should be moved to the backend sooner or later + qrcode = segno.make(kitem.url) + if image: + out = io.BytesIO() + if isinstance(image, Image.Image): + raise TypeError( + """When a QR Code is generated with an image as background, + its filepath must be a string""" + ) + qrcode.to_artistic( + background=image, target=out, scale=5, kind="jpeg", border=0 + ) + avatar = Image.open(out) + else: + avatar = qrcode.to_pil(scale=5, border=0) + if image and not make_qr: + if isinstance(image, str): + avatar = Image.open(image) + else: + avatar = image + if not image and not make_qr: + raise RuntimeError( + "Cannot generate avator. Neither `include_qr` or `file` are specified." + ) + return avatar + + +def _get_avatar(kitem: "KItem") -> Image.Image: + response = _perform_request(f"api/knowledge/avatar/{kitem.id}", "get") + buffer = io.BytesIO(response.content) + return Image.open(buffer) + + +def _create_or_update_app_spec(app: "AppConfig", overwrite=False) -> None: + """Create app specfication""" + upload_file = {"def_file": io.StringIO(yaml.safe_dump(app.specification))} + response = _perform_request( + f"/api/knowledge/apps/argo/spec/{app.name}", + "post", + files=upload_file, + params={"overwrite": overwrite}, + ) + if not response.ok: + message = f"Something went wrong uploading app spec with name `{app.name}`: {response.text}" + raise RuntimeError(message) + return response.text + + +def _delete_app_spec(name: str) -> None: + """Delete app specfication""" + response = _perform_request( + f"/api/knowledge/apps/argo/spec/{name}", + "delete", + ) + if not response.ok: + message = f"Something went wrong deleting app spec with name `{name}`: {response.text}" + raise RuntimeError(message) + return response.text diff --git a/examples/basic_usage.ipynb b/examples/basic_usage.ipynb index 53575f5..d66b53c 100644 --- a/examples/basic_usage.ipynb +++ b/examples/basic_usage.ipynb @@ -22,11 +22,6 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "from pprint import pprint\n", - "\n", - "from dotenv import load_dotenv\n", - "\n", "from dsms import DSMS, KItem" ] }, @@ -43,13 +38,7 @@ "metadata": {}, "outputs": [], "source": [ - "#specify path to an arbitrary file\n", - "env = os.path.join(\"..\", \".env\")\n", - "\n", - "# start the session\n", - "load_dotenv(env)\n", - "\n", - "dsms = DSMS()" + "dsms = DSMS(env=\"../.env\")" ] }, { @@ -66,26 +55,6 @@ "We can see which kind of DSMS-object we own as a user:" ] }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.kitems" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -95,636 +64,6 @@ "The schema of the KItem itself is a JSON schema which is machine-readable and can be directly incorporated into [Swagger](https://swagger.io/tools/swagger-ui/)-supported APIs like e.g. [`FastAPI`](https://fastapi.tiangolo.com/)." ] }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'$defs': {'AdditionalProperties': {'description': 'Additional properties of',\n", - " 'properties': {'triggerUponUpload': {'default': False,\n", - " 'description': 'Whether '\n", - " 'the '\n", - " 'app '\n", - " 'should '\n", - " 'be '\n", - " 'triggered '\n", - " 'when '\n", - " 'a '\n", - " 'file '\n", - " 'is '\n", - " 'uploaded',\n", - " 'title': 'Triggeruponupload',\n", - " 'type': 'boolean'},\n", - " 'triggerUponUploadFileExtensions': {'anyOf': [{'items': {'type': 'string'},\n", - " 'type': 'array'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'File '\n", - " 'extensions '\n", - " 'for '\n", - " 'which '\n", - " 'the '\n", - " 'upload '\n", - " 'shall '\n", - " 'be '\n", - " 'triggered.',\n", - " 'title': 'Triggeruponuploadfileextensions'}},\n", - " 'title': 'AdditionalProperties',\n", - " 'type': 'object'},\n", - " 'Affiliation': {'description': 'Affiliation of a KItem.',\n", - " 'properties': {'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem ID '\n", - " 'related to '\n", - " 'the '\n", - " 'KPropertyItem',\n", - " 'title': 'Id'},\n", - " 'name': {'description': 'Name of the '\n", - " 'affiliation',\n", - " 'title': 'Name',\n", - " 'type': 'string'}},\n", - " 'required': ['name'],\n", - " 'title': 'Affiliation',\n", - " 'type': 'object'},\n", - " 'Annotation': {'description': 'KItem annotation model',\n", - " 'properties': {'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem ID '\n", - " 'related to the '\n", - " 'KPropertyItem',\n", - " 'title': 'Id'},\n", - " 'iri': {'description': 'IRI of the '\n", - " 'annotation',\n", - " 'title': 'Iri',\n", - " 'type': 'string'},\n", - " 'name': {'description': 'Name of the '\n", - " 'annotation',\n", - " 'title': 'Name',\n", - " 'type': 'string'},\n", - " 'namespace': {'description': 'Namespace '\n", - " 'of the '\n", - " 'annotation',\n", - " 'title': 'Namespace',\n", - " 'type': 'string'}},\n", - " 'required': ['iri', 'name', 'namespace'],\n", - " 'title': 'Annotation',\n", - " 'type': 'object'},\n", - " 'App': {'description': 'App of a KItem.',\n", - " 'properties': {'additionalProperties': {'anyOf': [{'$ref': '#/$defs/AdditionalProperties'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Additional '\n", - " 'properties '\n", - " 'related '\n", - " 'to '\n", - " 'the '\n", - " 'appilcation'},\n", - " 'description': {'anyOf': [{'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Description '\n", - " 'of the '\n", - " 'appilcation',\n", - " 'title': 'Description'},\n", - " 'executable': {'description': 'Name of the '\n", - " 'executable '\n", - " 'related to '\n", - " 'the app',\n", - " 'title': 'Executable',\n", - " 'type': 'string'},\n", - " 'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem ID related to '\n", - " 'the KPropertyItem',\n", - " 'title': 'Id'},\n", - " 'kitemAppId': {'anyOf': [{'type': 'integer'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'ID of the '\n", - " 'KItem App',\n", - " 'title': 'Kitemappid'},\n", - " 'tags': {'anyOf': [{'type': 'object'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Tags related to the '\n", - " 'appilcation',\n", - " 'title': 'Tags'},\n", - " 'title': {'anyOf': [{'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Title of the '\n", - " 'appilcation',\n", - " 'title': 'Title'}},\n", - " 'required': ['executable'],\n", - " 'title': 'App',\n", - " 'type': 'object'},\n", - " 'Attachment': {'description': 'Attachment uploaded by a certain '\n", - " 'user.',\n", - " 'properties': {'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem ID '\n", - " 'related to the '\n", - " 'KPropertyItem',\n", - " 'title': 'Id'},\n", - " 'name': {'description': 'File name of '\n", - " 'the '\n", - " 'attachment',\n", - " 'title': 'Name',\n", - " 'type': 'string'}},\n", - " 'required': ['name'],\n", - " 'title': 'Attachment',\n", - " 'type': 'object'},\n", - " 'Author': {'description': 'Author of a KItem.',\n", - " 'properties': {'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem ID related '\n", - " 'to the '\n", - " 'KPropertyItem',\n", - " 'title': 'Id'},\n", - " 'userId': {'description': 'ID of the DSMS '\n", - " 'User',\n", - " 'format': 'uuid',\n", - " 'title': 'Userid',\n", - " 'type': 'string'}},\n", - " 'required': ['userId'],\n", - " 'title': 'Author',\n", - " 'type': 'object'},\n", - " 'Column': {'description': 'Column of an HDF5 data frame',\n", - " 'properties': {'columnId': {'description': 'Column ID in '\n", - " 'the data '\n", - " 'frame',\n", - " 'title': 'Columnid',\n", - " 'type': 'integer'},\n", - " 'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem ID related '\n", - " 'to the '\n", - " 'KPropertyItem',\n", - " 'title': 'Id'},\n", - " 'name': {'description': 'Name of the '\n", - " 'column in the '\n", - " 'data series.',\n", - " 'title': 'Name',\n", - " 'type': 'string'}},\n", - " 'required': ['columnId', 'name'],\n", - " 'title': 'Column',\n", - " 'type': 'object'},\n", - " 'ContactInfo': {'description': 'Contact info',\n", - " 'properties': {'email': {'description': 'EMail of '\n", - " 'the '\n", - " 'contact '\n", - " 'person',\n", - " 'title': 'Email',\n", - " 'type': 'string'},\n", - " 'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem ID '\n", - " 'related to '\n", - " 'the '\n", - " 'KPropertyItem',\n", - " 'title': 'Id'},\n", - " 'name': {'description': 'Name of the '\n", - " 'contact '\n", - " 'person',\n", - " 'title': 'Name',\n", - " 'type': 'string'},\n", - " 'userId': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'User ID '\n", - " 'of the '\n", - " 'contact '\n", - " 'person',\n", - " 'title': 'Userid'}},\n", - " 'required': ['name', 'email'],\n", - " 'title': 'ContactInfo',\n", - " 'type': 'object'},\n", - " 'ExternalLink': {'description': 'External link of a KItem.',\n", - " 'properties': {'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem ID '\n", - " 'related to '\n", - " 'the '\n", - " 'KPropertyItem',\n", - " 'title': 'Id'},\n", - " 'label': {'description': 'Label of '\n", - " 'the '\n", - " 'external '\n", - " 'link',\n", - " 'title': 'Label',\n", - " 'type': 'string'},\n", - " 'url': {'description': 'URL of the '\n", - " 'external '\n", - " 'link',\n", - " 'format': 'uri',\n", - " 'minLength': 1,\n", - " 'title': 'Url',\n", - " 'type': 'string'}},\n", - " 'required': ['label', 'url'],\n", - " 'title': 'ExternalLink',\n", - " 'type': 'object'},\n", - " 'KItem': {'additionalProperties': False,\n", - " 'description': 'Knowledge Item of the DSMS.',\n", - " 'properties': {'affiliations': {'default': [],\n", - " 'description': 'Affiliations '\n", - " 'related '\n", - " 'to a '\n", - " 'KItem.',\n", - " 'items': {'$ref': '#/$defs/Affiliation'},\n", - " 'title': 'Affiliations',\n", - " 'type': 'array'},\n", - " 'annotations': {'default': [],\n", - " 'description': 'Annotations '\n", - " 'of the '\n", - " 'KItem',\n", - " 'items': {'$ref': '#/$defs/Annotation'},\n", - " 'title': 'Annotations',\n", - " 'type': 'array'},\n", - " 'attachments': {'default': [],\n", - " 'description': 'File '\n", - " 'attachements '\n", - " 'of the '\n", - " 'DSMS',\n", - " 'items': {'anyOf': [{'$ref': '#/$defs/Attachment'},\n", - " {'type': 'string'}]},\n", - " 'title': 'Attachments',\n", - " 'type': 'array'},\n", - " 'authors': {'default': [],\n", - " 'description': 'Authorship of '\n", - " 'the KItem.',\n", - " 'items': {'anyOf': [{'$ref': '#/$defs/Author'},\n", - " {'type': 'string'}]},\n", - " 'title': 'Authors',\n", - " 'type': 'array'},\n", - " 'avatar_exists': {'anyOf': [{'type': 'boolean'},\n", - " {'type': 'null'}],\n", - " 'default': False,\n", - " 'description': 'Whether '\n", - " 'the '\n", - " 'KItem '\n", - " 'holds an '\n", - " 'avatar '\n", - " 'or not.',\n", - " 'title': 'Avatar Exists'},\n", - " 'contacts': {'default': [],\n", - " 'description': 'Whether the '\n", - " 'KItem holds '\n", - " 'any contact '\n", - " 'information.',\n", - " 'items': {'$ref': '#/$defs/ContactInfo'},\n", - " 'title': 'Contacts',\n", - " 'type': 'array'},\n", - " 'created_at': {'anyOf': [{'type': 'string'},\n", - " {'format': 'date-time',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Time and '\n", - " 'date when '\n", - " 'the KItem '\n", - " 'was '\n", - " 'created.',\n", - " 'title': 'Created At'},\n", - " 'custom_properties': {'anyOf': [{},\n", - " {'type': 'null'}],\n", - " 'default': {},\n", - " 'description': 'Custom '\n", - " 'properties '\n", - " 'associated '\n", - " 'to '\n", - " 'the '\n", - " 'KItem',\n", - " 'title': 'Custom '\n", - " 'Properties'},\n", - " 'external_links': {'default': [],\n", - " 'description': 'External '\n", - " 'links '\n", - " 'related '\n", - " 'to the '\n", - " 'KItem',\n", - " 'items': {'$ref': '#/$defs/ExternalLink'},\n", - " 'title': 'External '\n", - " 'Links',\n", - " 'type': 'array'},\n", - " 'hdf5': {'anyOf': [{'items': {'$ref': '#/$defs/Column'},\n", - " 'type': 'array'},\n", - " {'additionalProperties': {'anyOf': [{'items': {},\n", - " 'type': 'array'},\n", - " {'type': 'object'}]},\n", - " 'type': 'object'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'HDF5 interface.',\n", - " 'title': 'Hdf5'},\n", - " 'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'description': 'ID of the KItem',\n", - " 'title': 'Id'},\n", - " 'kitem_apps': {'default': [],\n", - " 'description': 'Apps '\n", - " 'related to '\n", - " 'the KItem.',\n", - " 'items': {'$ref': '#/$defs/App'},\n", - " 'title': 'Kitem Apps',\n", - " 'type': 'array'},\n", - " 'ktype': {'anyOf': [{'$ref': '#/$defs/KType'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KType of the '\n", - " 'KItem'},\n", - " 'ktype_id': {'anyOf': [{'description': 'Create '\n", - " 'a '\n", - " 'collection '\n", - " 'of '\n", - " 'name/value '\n", - " 'pairs.\\n'\n", - " '\\n'\n", - " 'Example '\n", - " 'enumeration:\\n'\n", - " '\\n'\n", - " '>>> '\n", - " 'class '\n", - " 'Color(Enum):\\n'\n", - " '... '\n", - " 'RED '\n", - " '= '\n", - " '1\\n'\n", - " '... '\n", - " 'BLUE '\n", - " '= '\n", - " '2\\n'\n", - " '... '\n", - " 'GREEN '\n", - " '= '\n", - " '3\\n'\n", - " '\\n'\n", - " 'Access '\n", - " 'them '\n", - " 'by:\\n'\n", - " '\\n'\n", - " '- '\n", - " 'attribute '\n", - " 'access:\\n'\n", - " '\\n'\n", - " ' '\n", - " '>>> '\n", - " 'Color.RED\\n'\n", - " ' '\n", - " '\\n'\n", - " '\\n'\n", - " '- '\n", - " 'value '\n", - " 'lookup:\\n'\n", - " '\\n'\n", - " ' '\n", - " '>>> '\n", - " 'Color(1)\\n'\n", - " ' '\n", - " '\\n'\n", - " '\\n'\n", - " '- '\n", - " 'name '\n", - " 'lookup:\\n'\n", - " '\\n'\n", - " ' '\n", - " '>>> '\n", - " \"Color['RED']\\n\"\n", - " ' '\n", - " '\\n'\n", - " '\\n'\n", - " 'Enumerations '\n", - " 'can '\n", - " 'be '\n", - " 'iterated '\n", - " 'over, '\n", - " 'and '\n", - " 'know '\n", - " 'how '\n", - " 'many '\n", - " 'members '\n", - " 'they '\n", - " 'have:\\n'\n", - " '\\n'\n", - " '>>> '\n", - " 'len(Color)\\n'\n", - " '3\\n'\n", - " '\\n'\n", - " '>>> '\n", - " 'list(Color)\\n'\n", - " '[, '\n", - " ', '\n", - " ']\\n'\n", - " '\\n'\n", - " 'Methods '\n", - " 'can '\n", - " 'be '\n", - " 'added '\n", - " 'to '\n", - " 'enumerations, '\n", - " 'and '\n", - " 'members '\n", - " 'can '\n", - " 'have '\n", - " 'their '\n", - " 'own\\n'\n", - " 'attributes '\n", - " '-- '\n", - " 'see '\n", - " 'the '\n", - " 'documentation '\n", - " 'for '\n", - " 'details.',\n", - " 'enum': [],\n", - " 'title': 'Enum'},\n", - " {'type': 'string'}],\n", - " 'description': 'Type ID of '\n", - " 'the KItem',\n", - " 'title': 'Ktype Id'},\n", - " 'linked_kitems': {'default': [],\n", - " 'description': 'KItems '\n", - " 'linked '\n", - " 'to the '\n", - " 'current '\n", - " 'KItem.',\n", - " 'items': {'anyOf': [{'$ref': '#/$defs/LinkedKItem'},\n", - " {'$ref': '#/$defs/KItem'}]},\n", - " 'title': 'Linked Kitems',\n", - " 'type': 'array'},\n", - " 'name': {'description': 'Human readable '\n", - " 'name of the '\n", - " 'KContext.dsms',\n", - " 'title': 'Name',\n", - " 'type': 'string'},\n", - " 'slug': {'anyOf': [{'minLength': 4,\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Slug of the '\n", - " 'KContext.dsms',\n", - " 'title': 'Slug'},\n", - " 'summary': {'anyOf': [{'type': 'string'},\n", - " {'$ref': '#/$defs/Summary'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Human readable '\n", - " 'summary text '\n", - " 'of the KItem.',\n", - " 'title': 'Summary'},\n", - " 'updated_at': {'anyOf': [{'type': 'string'},\n", - " {'format': 'date-time',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Time and '\n", - " 'date when '\n", - " 'the KItem '\n", - " 'was '\n", - " 'updated.',\n", - " 'title': 'Updated At'},\n", - " 'user_groups': {'default': [],\n", - " 'description': 'User '\n", - " 'groups '\n", - " 'able to '\n", - " 'access the '\n", - " 'KItem.',\n", - " 'items': {'$ref': '#/$defs/UserGroup'},\n", - " 'title': 'User Groups',\n", - " 'type': 'array'}},\n", - " 'required': ['name', 'ktype_id'],\n", - " 'title': 'KItem',\n", - " 'type': 'object'},\n", - " 'KType': {'description': 'Knowledge type of the knowledge item.',\n", - " 'properties': {'data_schema': {'anyOf': [{},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'OpenAPI '\n", - " 'schema of '\n", - " 'the KItem.',\n", - " 'title': 'Data Schema'},\n", - " 'form_data': {'anyOf': [{},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Form data of '\n", - " 'the KItem.',\n", - " 'title': 'Form Data'},\n", - " 'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'string'}],\n", - " 'description': 'ID of the KType.',\n", - " 'title': 'Id'},\n", - " 'name': {'anyOf': [{'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Human readable '\n", - " 'name of the '\n", - " 'KType.',\n", - " 'title': 'Name'}},\n", - " 'required': ['id'],\n", - " 'title': 'KType',\n", - " 'type': 'object'},\n", - " 'LinkedKItem': {'description': 'Data model of a linked KItem',\n", - " 'properties': {'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'ID of the '\n", - " 'KItem to be '\n", - " 'linked',\n", - " 'title': 'Id'},\n", - " 'sourceId': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'Source '\n", - " 'ID of '\n", - " 'the '\n", - " 'KItem',\n", - " 'title': 'Sourceid'}},\n", - " 'title': 'LinkedKItem',\n", - " 'type': 'object'},\n", - " 'Summary': {'additionalProperties': False,\n", - " 'description': 'Model for the custom properties of the '\n", - " 'KItem',\n", - " 'properties': {'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'title': 'Id'},\n", - " 'kitem': {'anyOf': [{}, {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem related '\n", - " 'to the summary',\n", - " 'title': 'Kitem'},\n", - " 'text': {'description': 'Summary text of '\n", - " 'the KItem',\n", - " 'title': 'Text',\n", - " 'type': 'string'}},\n", - " 'required': ['text'],\n", - " 'title': 'Summary',\n", - " 'type': 'object'},\n", - " 'UserGroup': {'description': 'Users groups related to a KItem.',\n", - " 'properties': {'groupId': {'description': 'ID of the '\n", - " 'user group',\n", - " 'title': 'Groupid',\n", - " 'type': 'string'},\n", - " 'id': {'anyOf': [{'format': 'uuid',\n", - " 'type': 'string'},\n", - " {'type': 'null'}],\n", - " 'default': None,\n", - " 'description': 'KItem ID '\n", - " 'related to the '\n", - " 'KPropertyItem',\n", - " 'title': 'Id'},\n", - " 'name': {'description': 'Name of the '\n", - " 'user group',\n", - " 'title': 'Name',\n", - " 'type': 'string'}},\n", - " 'required': ['name', 'groupId'],\n", - " 'title': 'UserGroup',\n", - " 'type': 'object'}},\n", - " 'allOf': [{'$ref': '#/$defs/KItem'}]}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\Anaconda3\\envs\\sdk\\Lib\\site-packages\\pydantic\\json_schema.py:2099: PydanticJsonSchemaWarning: Default value is not JSON serializable; excluding default from JSON schema [non-serializable-default]\n", - " warnings.warn(message, PydanticJsonSchemaWarning)\n" - ] - } - ], - "source": [ - "pprint(KItem.model_json_schema())" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -734,7 +73,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -742,11 +81,15 @@ "output_type": "stream", "text": [ "KTypes.Organization\n", - "KTypes.Expert\n", "KTypes.App\n", + "KTypes.Dataset\n", "KTypes.DatasetCatalog\n", - "KTypes.TestSeries\n", - "KTypes.TestMatthias\n" + "KTypes.Expert\n", + "KTypes.Test\n", + "KTypes.Specimen\n", + "KTypes.Batch\n", + "KTypes.Resource\n", + "KTypes.TestingMachine\n" ] } ], @@ -771,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -781,11 +124,13 @@ "\n", "\tname = foo123, \n", "\n", - "\tid = 23fcb340-dec7-483b-9236-b30d8f8ea01c, \n", + "\tid = 617cd64f-1a4d-48d5-aed6-ff9d9e3ddb20, \n", "\n", - "\tktype_id = KTypes.DatasetCatalog, \n", + "\tktype_id = KTypes.Dataset, \n", "\n", - "\tslug = foo123, \n", + "\tin_backend = False, \n", + "\n", + "\tslug = foo123-617cd64f, \n", "\n", "\tannotations = [], \n", "\n", @@ -813,13 +158,17 @@ "\n", "\tuser_groups = [], \n", "\n", - "\tcustom_properties = CustomProperties(content={'foo': 'bar'}), \n", + "\tcustom_properties = {\n", + "\t\tfoo: bar\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", "\n", - "\thdf5 = None\n", + "\trdf_exists = False\n", ")" ] }, - "execution_count": 6, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -827,7 +176,7 @@ "source": [ "item = KItem(\n", " name=\"foo123\",\n", - " ktype_id=dsms.ktypes.DatasetCatalog,\n", + " ktype_id=dsms.ktypes.Dataset,\n", " custom_properties={\"foo\": \"bar\"},\n", ")\n", "\n", @@ -843,11 +192,23 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'https://bue.materials-data.space/knowledge/dataset/foo123-617cd64f'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "dsms.commit()" + "dsms.commit()\n", + "item.url" ] }, { @@ -859,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -869,11 +230,13 @@ "\n", "\tname = foo123, \n", "\n", - "\tid = 23fcb340-dec7-483b-9236-b30d8f8ea01c, \n", + "\tid = 617cd64f-1a4d-48d5-aed6-ff9d9e3ddb20, \n", "\n", - "\tktype_id = dataset-catalog, \n", + "\tktype_id = dataset, \n", "\n", - "\tslug = foo123, \n", + "\tin_backend = True, \n", + "\n", + "\tslug = foo123-617cd64f, \n", "\n", "\tannotations = [], \n", "\n", @@ -884,16 +247,18 @@ "\taffiliations = [], \n", "\n", "\tauthors = [\n", - "\t\tAuthor(user_id=4440f8c8-f443-4114-a7a7-d8e2a6b5d776)\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", "\t], \n", "\n", "\tavatar_exists = False, \n", "\n", "\tcontacts = [], \n", "\n", - "\tcreated_at = 2024-03-06 10:12:39.377020, \n", + "\tcreated_at = 2024-08-20 08:01:50.536406, \n", "\n", - "\tupdated_at = 2024-03-06 10:12:39.377020, \n", + "\tupdated_at = 2024-08-20 08:01:50.536406, \n", "\n", "\texternal_links = [], \n", "\n", @@ -903,13 +268,17 @@ "\n", "\tuser_groups = [], \n", "\n", - "\tcustom_properties = CustomProperties(content={'foo': 'bar'}), \n", + "\tcustom_properties = {\n", + "\t\tfoo: bar\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", "\n", - "\thdf5 = None\n", + "\trdf_exists = False\n", ")" ] }, - "execution_count": 8, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -918,74 +287,6 @@ "item" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And the list of our KItems in the DSMS has been updated as well:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[KItem(\n", - " \n", - " \tname = foo123, \n", - " \n", - " \tid = 23fcb340-dec7-483b-9236-b30d8f8ea01c, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = foo123, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=4440f8c8-f443-4114-a7a7-d8e2a6b5d776)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-06 10:12:39.377020, \n", - " \n", - " \tupdated_at = 2024-03-06 10:12:39.377020, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={'foo': 'bar'}), \n", - " \n", - " \thdf5 = None\n", - " )]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.kitems" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -1007,28 +308,19 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "# specify the path to any arbitrary file to be uploaded\n", - "file = os.path.join(\"..\", \"README.md\")\n", - "\n", "item.name = \"foobar\"\n", - "item.custom_properties.update({\"foobar\": \"foobar\"})\n", - "item.attachments.append({\"name\": file})\n", - "item.annotations.append(\n", - " {\n", - " \"iri\": \"www.example.org/foo\",\n", - " \"name\": \"example class\",\n", - " \"namespace\": \"www.example.org\",\n", - " }\n", - ")\n", + "item.custom_properties.foobar = \"foobar\"\n", + "item.attachments.append(\"../README.md\")\n", + "item.annotations.append(\"www.example.org/foo\")\n", "item.external_links.append(\n", " {\"url\": \"http://example.org\", \"label\": \"example link\"}\n", ")\n", "item.contacts.append({\"name\": \"foo\", \"email\": \"foo@bar.mail\"})\n", - "item.affiliations.append({\"name\": \"foobar team\"})\n", + "item.affiliations.append(\"foobar team\")\n", "item.user_groups.append({\"name\": \"foogroup\", \"group_id\": \"123\"})" ] }, @@ -1041,13 +333,113 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "dsms.commit()" ] }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KItem(\n", + "\n", + "\tname = foobar, \n", + "\n", + "\tid = 617cd64f-1a4d-48d5-aed6-ff9d9e3ddb20, \n", + "\n", + "\tktype_id = dataset, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = foo123-617cd64f, \n", + "\n", + "\tannotations = [\n", + "\t\t{\n", + "\t\t\tiri: www.example.org/foo,\n", + "\t\t\tname: foo,\n", + "\t\t\tnamespace: www.example.org,\n", + "\t\t\tdescription: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tattachments = [\n", + "\t\t{\n", + "\t\t\tname: README.md\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [\n", + "\t\t{\n", + "\t\t\tname: foobar team\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [\n", + "\t\t{\n", + "\t\t\tname: foo,\n", + "\t\t\temail: foo@bar.mail,\n", + "\t\t\tuser_id: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcreated_at = 2024-08-20 08:01:50.536406, \n", + "\n", + "\tupdated_at = 2024-08-20 08:01:54.745601, \n", + "\n", + "\texternal_links = [\n", + "\t\t{\n", + "\t\t\tlabel: example link,\n", + "\t\t\turl: http://example.org/\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [\n", + "\t\t{\n", + "\t\t\tname: foogroup,\n", + "\t\t\tgroup_id: 123\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tfoo: bar\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1064,14 +456,16 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\t\t\t Downloaded file: Attachment(name=README.md)\n", + "\t\t\t Downloaded file: {\n", + "\t\t\tname: README.md\n", + "\t\t}\n", "|------------------------------------Beginning of file------------------------------------|\n", "# DSMS-SDK\n", "Python SDK core-package for interacting with the Dataspace Management System (DSMS)\n", @@ -1105,7 +499,6 @@ "\n", "For the basic usage, please have a look on the Jupyter Notebook under `examples/basic_usage.ipynb`. This tutorial provides a basic overview of using the dsms package to interact with Knowledge Items.\n", "\n", - "\n", "## Disclaimer\n", "\n", "Copyright (c) 2014-2024, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. acting on behalf of its Fraunhofer IWM.\n", @@ -1148,26 +541,28 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "UserGroup(name=foogroup, group_id=123)" + "{\n", + "\t\t\tiri: www.example.org/foo,\n", + "\t\t\tname: foo,\n", + "\t\t\tnamespace: www.example.org,\n", + "\t\t\tdescription: None\n", + "\t\t}" ] }, - "execution_count": 13, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "item.attachments.pop(0)\n", - "item.annotations.pop(0)\n", - "item.external_links.pop(0)\n", - "item.contacts.pop(0)\n", - "item.user_groups.pop(0)" + "item.annotations.pop(0)" ] }, { @@ -1179,11 +574,11 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ - "item.affiliations = []" + "item.user_groups = []" ] }, { @@ -1193,9 +588,25 @@ "See the changes:" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Send the changes to the DSMS with the `commit`-method:" + ] + }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -1205,11 +616,13 @@ "\n", "\tname = foobar, \n", "\n", - "\tid = 23fcb340-dec7-483b-9236-b30d8f8ea01c, \n", + "\tid = 617cd64f-1a4d-48d5-aed6-ff9d9e3ddb20, \n", + "\n", + "\tktype_id = dataset, \n", "\n", - "\tktype_id = dataset-catalog, \n", + "\tin_backend = True, \n", "\n", - "\tslug = foo123, \n", + "\tslug = foo123-617cd64f, \n", "\n", "\tannotations = [], \n", "\n", @@ -1217,21 +630,38 @@ "\n", "\tlinked_kitems = [], \n", "\n", - "\taffiliations = [], \n", + "\taffiliations = [\n", + "\t\t{\n", + "\t\t\tname: foobar team\n", + "\t\t}\n", + "\t], \n", "\n", "\tauthors = [\n", - "\t\tAuthor(user_id=4440f8c8-f443-4114-a7a7-d8e2a6b5d776)\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", "\t], \n", "\n", "\tavatar_exists = False, \n", "\n", - "\tcontacts = [], \n", + "\tcontacts = [\n", + "\t\t{\n", + "\t\t\tname: foo,\n", + "\t\t\temail: foo@bar.mail,\n", + "\t\t\tuser_id: None\n", + "\t\t}\n", + "\t], \n", "\n", - "\tcreated_at = 2024-03-06 10:12:39.377020, \n", + "\tcreated_at = 2024-08-20 08:01:50.536406, \n", "\n", - "\tupdated_at = 2024-03-06 10:13:06.308318, \n", + "\tupdated_at = 2024-08-20 08:01:54.745601, \n", "\n", - "\texternal_links = [], \n", + "\texternal_links = [\n", + "\t\t{\n", + "\t\t\tlabel: example link,\n", + "\t\t\turl: http://example.org/\n", + "\t\t}\n", + "\t], \n", "\n", "\tkitem_apps = [], \n", "\n", @@ -1239,13 +669,17 @@ "\n", "\tuser_groups = [], \n", "\n", - "\tcustom_properties = CustomProperties(content={'foobar': 'foobar'}), \n", + "\tcustom_properties = {\n", + "\t\tfoo: bar\n", + "\t}, \n", "\n", - "\thdf5 = None\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", ")" ] }, - "execution_count": 15, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -1258,877 +692,161 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Send the changes to the DSMS with the `commit`-method:" + "However, we can also delete the whole KItem from the DSMS by applying the `del`-operator to the `dsms`-object with the individual `KItem`-object:" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ - "dsms.commit()" + "del dsms[item]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "However, we can also delete the whole KItem from the DSMS by applying the `del`-operator to the `dsms`-object with the individual `KItem`-object:" + "Commit the changes:" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ - "del dsms[item]" + "dsms.commit()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Commit the changes:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "dsms.commit()" + "### 5: Search for KItems" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "As we can see know, our KItem has been removed from our inventory:" + "In the last unit of this tutorial, we would like to search for specfic KItems we created in the DSMS.\n", + "\n", + "For this purpose, we will firstly create some KItems and apply the `search`-method on the `DSMS`-object later on in order to find them again in the DSMS.\n", + "\n", + "We also wnat to demonstrate here, that we can link KItems to each other in order to find e.g. a related item of type `DatasetCatalog`. For this strategy, we are using the `linked_kitems`-attribute and the `id` of the item which we would like to link.\n", + "\n", + "The procedure looks like this:" ] }, { "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.kitems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5: Search for KItems" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the last unit of this tutorial, we would like to search for specfic KItems we created in the DSMS.\n", - "\n", - "For this purpose, we will firstly create some KItems and apply the `search`-method on the `DSMS`-object later on in order to find them again in the DSMS.\n", - "\n", - "We also wnat to demonstrate here, that we can link KItems to each other in order to find e.g. a related item of type `DatasetCatalog`. For this strategy, we are using the `linked_kitems`-attribute and the `id` of the item which we would like to link.\n", - "\n", - "The procedure looks like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 20, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "item = KItem(\n", " name=\"foo 1\",\n", " ktype_id=dsms.ktypes.DatasetCatalog\n", - ")\n", - "\n", - "item2 = KItem(\n", - " name=\"foo 2\",\n", - " ktype_id=dsms.ktypes.Organization,\n", - " linked_kitems=[item],\n", - " annotations=[\n", - " {\n", - " \"iri\": \"www.example.org/foo\",\n", - " \"name\": \"foo\",\n", - " \"namespace\": \"www.example.org\",\n", - " }\n", - " ],\n", - ")\n", - "item3 = KItem(\n", - " name=\"foo 3\", \n", - " ktype_id=dsms.ktypes.Organization\n", - ")\n", - "item4 = KItem(\n", - " name=\"foo 4\",\n", - " ktype_id=dsms.ktypes.Organization,\n", - " annotations=[\n", - " {\n", - " \"iri\": \"www.example.org/bar\",\n", - " \"name\": \"bar\",\n", - " \"namespace\": \"https://www.example.org\",\n", - " }\n", - " ],\n", - ")\n", - "\n", - "dsms.commit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, we are apply to search for e.g. kitems of type `DatasetCatalog`:" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[KItem(\n", - " \n", - " \tname = excel_component_test, \n", - " \n", - " \tid = 87eed190-3979-4c5f-bae5-2dbad25f2f12, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = excel_component_test, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 16:45:15.506823, \n", - " \n", - " \tupdated_at = 2024-03-05 16:45:15.506823, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = csv_ebsd, \n", - " \n", - " \tid = c113be0f-77cf-447e-b14f-a84c893cec23, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = csv_ebsd, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=11533_FS_Scan1_830x830_100x.ang), \n", - " \t\tAttachment(name=DP800_111_WR00_500x_150x150um.ang), \n", - " \t\tAttachment(name=AA6014_M_REM_WR00_196_merged_cleaned.ang), \n", - " \t\tAttachment(name=DP800_112_WR90_500x_150x150.ang), \n", - " \t\tAttachment(name=dx56_mitte_quer_850x850_step1-5.ang), \n", - " \t\tAttachment(name=AA6014_M_REM_WR90_197_merged_cleaned.ang), \n", - " \t\tAttachment(name=dx56_mitte_laengs_850x850_step1-5.ang)\n", - " \t], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 16:45:48.101341, \n", - " \n", - " \tupdated_at = 2024-03-05 16:45:48.101341, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = csv_bulgetest, \n", - " \n", - " \tid = e64c512a-c2e5-428d-bc85-5d0d9c2d68c7, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = csv_bulgetest, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=DX56_C_BLG_WR90_42.TXT), \n", - " \t\tAttachment(name=AA6014_O_BLG_WR90_241.TXT), \n", - " \t\tAttachment(name=DP800_H_BLG_WR90_140.TXT), \n", - " \t\tAttachment(name=DX56_D_BLG_WR90_56.TXT), \n", - " \t\tAttachment(name=AFZ1-BU-S8.TXT), \n", - " \t\tAttachment(name=AFZ-BU-S1.TXT), \n", - " \t\tAttachment(name=AA6014_N_BLG_WR90_218.TXT), \n", - " \t\tAttachment(name=DP800_G_BLG_WR90_126.TXT), \n", - " \t\tAttachment(name=AFZ-BU-S7.TXT), \n", - " \t\tAttachment(name=AA6014_M_BLG_WR90_195.TXT), \n", - " \t\tAttachment(name=DX56_B_BLG_WR90_28.TXT), \n", - " \t\tAttachment(name=DP800_F_BLG_WR90_108.TXT)\n", - " \t], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 16:46:41.767008, \n", - " \n", - " \tupdated_at = 2024-03-05 16:46:41.767008, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = excel_shear_tensile_test, \n", - " \n", - " \tid = e9c4bb04-9482-458d-9d12-c2ad8f214930, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = excel_shear_tensile_test, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 16:47:27.228060, \n", - " \n", - " \tupdated_at = 2024-03-05 16:47:27.228060, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = excel_tensile_test, \n", - " \n", - " \tid = e1f1b94c-5bd6-44c7-9de4-1d01650ac7b8, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = excel_tensile_test, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 16:48:01.936787, \n", - " \n", - " \tupdated_at = 2024-03-05 16:48:01.936787, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [\n", - " \t\tApp(kitem_app_id=105, executable=excel_tensile_test/excel_tensile_test.ipynb, title=excel_tensile_test, description=excel_tensile_test, tags=None, additional_properties=triggerUponUpload=True triggerUponUploadFileExtensions=['.xls', '.xlsm'])\n", - " \t], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = csv_tensile_test, \n", - " \n", - " \tid = 90207798-d6b2-47fb-b208-f7b8f771de0d, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = csv_tensile_test, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=DX56_D_FZ2_WR00_43.TXT)\n", - " \t], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 17:08:20.345033, \n", - " \n", - " \tupdated_at = 2024-03-05 17:08:20.345033, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [\n", - " \t\tApp(kitem_app_id=107, executable=csv_tensile_test/csv_tensile_test.ipynb, title=csv_tensile_test, description=csv_tensile_test, tags=None, additional_properties=triggerUponUpload=True triggerUponUploadFileExtensions=['.TXT']), \n", - " \t\tApp(kitem_app_id=108, executable=csv_tensile_test/csv_tensile_test.ipynb, title=csv_tensile_test, description=csv_tensile_test, tags=None, additional_properties=triggerUponUpload=True triggerUponUploadFileExtensions=['.TXT'])\n", - " \t], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = [\n", - " \t\tColumn(column_id=0, name=Prüfzeit), \n", - " \t\tColumn(column_id=1, name=Standardkraft), \n", - " \t\tColumn(column_id=2, name=Traversenweg absolut), \n", - " \t\tColumn(column_id=3, name=Standardweg), \n", - " \t\tColumn(column_id=4, name=Breitenänderung), \n", - " \t\tColumn(column_id=5, name=Dehnung)\n", - " \t]\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = csv_tensile_test_f2, \n", - " \n", - " \tid = c9203684-f1ce-4d23-92f9-478a002b7440, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = csv_tensile_test_f2, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=DP800_F_FZ2_WR15_97.TXT)\n", - " \t], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 17:08:27.579333, \n", - " \n", - " \tupdated_at = 2024-03-05 17:08:27.579333, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [\n", - " \t\tApp(kitem_app_id=109, executable=csv_tensile_test_f2/csv_tensile_test_f2.ipynb, title=csv_tensile_test_f2, description=csv_tensile_test_f2, tags=None, additional_properties=triggerUponUpload=True triggerUponUploadFileExtensions=['.TXT'])\n", - " \t], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = material constants, \n", - " \n", - " \tid = ea0bb87b-3a26-48be-84e1-bec9b99a1204, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = material_constants, \n", - " \n", - " \tannotations = [\n", - " \t\tAnnotation(iri=https://w3id.org/steel/ProcessOntology/MaterialCard, name=MaterialCard, namespace=https://w3id.org/steel/ProcessOntology)\n", - " \t], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [\n", - " \t\tLinkedKItem(id=72bc54cf-8138-4ac3-8586-14afe5bc6b7f, source_id=ea0bb87b-3a26-48be-84e1-bec9b99a1204), \n", - " \t\tLinkedKItem(id=253d69c1-606c-4721-832e-1caed961e8ef, source_id=ea0bb87b-3a26-48be-84e1-bec9b99a1204)\n", - " \t], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=8f40087b-16d6-49ba-a8ac-2bd70b8a2308)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-02-27 05:31:47.433035, \n", - " \n", - " \tupdated_at = 2024-02-27 05:31:47.433035, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={'material constants': {'C11': 226000.0, 'C12': 140000.0, 'C44': 116000.0, 'shear_rate_ref': 0.001, 'rate_sens': 0.0125, 'tau0': 62.1930648, 'tau1': 48.1881715, 'theta0': 356.842243, 'theta1': 37.4360252, 'q1': 1.4, 'q2': 1.4}}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = orientations, \n", - " \n", - " \tid = 5386bf18-533a-4976-8481-d54638f07a91, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = orientations, \n", - " \n", - " \tannotations = [\n", - " \t\tAnnotation(iri=https://w3id.org/steel/ProcessOntology/GrainOrientation, name=GrainOrientation, namespace=https://w3id.org/steel/ProcessOntology)\n", - " \t], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=11533_FS_Scan1_830x830_100x.ang)\n", - " \t], \n", - " \n", - " \tlinked_kitems = [\n", - " \t\tLinkedKItem(id=72bc54cf-8138-4ac3-8586-14afe5bc6b7f, source_id=5386bf18-533a-4976-8481-d54638f07a91), \n", - " \t\tLinkedKItem(id=253d69c1-606c-4721-832e-1caed961e8ef, source_id=5386bf18-533a-4976-8481-d54638f07a91)\n", - " \t], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=8f40087b-16d6-49ba-a8ac-2bd70b8a2308)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-02-27 05:31:47.498209, \n", - " \n", - " \tupdated_at = 2024-02-27 05:31:47.498209, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = rve_1000_grains_geofile, \n", - " \n", - " \tid = 13e57827-c5fe-413e-90dd-bb95ec5c1d3a, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = rve_1000_grains_geofile, \n", - " \n", - " \tannotations = [\n", - " \t\tAnnotation(iri=https://w3id.org/steel/ProcessOntology/Geometry, name=Geometry, namespace=https://w3id.org/steel/ProcessOntology)\n", - " \t], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=mesh_40x40x40_grains_1000_aspect_ratio_1_6.geo)\n", - " \t], \n", - " \n", - " \tlinked_kitems = [\n", - " \t\tLinkedKItem(id=253d69c1-606c-4721-832e-1caed961e8ef, source_id=13e57827-c5fe-413e-90dd-bb95ec5c1d3a)\n", - " \t], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=8f40087b-16d6-49ba-a8ac-2bd70b8a2308)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-02-27 05:31:47.588776, \n", - " \n", - " \tupdated_at = 2024-02-27 05:31:47.588776, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = foo 1, \n", - " \n", - " \tid = 8377f120-d558-4cc2-93d4-582bb01fac11, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = foo1, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [\n", - " \t\tLinkedKItem(id=5dacf60c-6a62-466a-a85e-9f3ce422cefa, source_id=8377f120-d558-4cc2-93d4-582bb01fac11)\n", - " \t], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=4440f8c8-f443-4114-a7a7-d8e2a6b5d776)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-06 10:13:29.213578, \n", - " \n", - " \tupdated_at = 2024-03-06 10:13:29.213578, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " )]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dsms.search(ktypes=[dsms.ktypes.DatasetCatalog])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "... and for all of type `Organization` and `DatasetCatalog`:" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[KItem(\n", - " \n", - " \tname = excel_component_test, \n", - " \n", - " \tid = 87eed190-3979-4c5f-bae5-2dbad25f2f12, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = excel_component_test, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 16:45:15.506823, \n", - " \n", - " \tupdated_at = 2024-03-05 16:45:15.506823, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = csv_ebsd, \n", - " \n", - " \tid = c113be0f-77cf-447e-b14f-a84c893cec23, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = csv_ebsd, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=11533_FS_Scan1_830x830_100x.ang), \n", - " \t\tAttachment(name=DP800_111_WR00_500x_150x150um.ang), \n", - " \t\tAttachment(name=AA6014_M_REM_WR00_196_merged_cleaned.ang), \n", - " \t\tAttachment(name=DP800_112_WR90_500x_150x150.ang), \n", - " \t\tAttachment(name=dx56_mitte_quer_850x850_step1-5.ang), \n", - " \t\tAttachment(name=AA6014_M_REM_WR90_197_merged_cleaned.ang), \n", - " \t\tAttachment(name=dx56_mitte_laengs_850x850_step1-5.ang)\n", - " \t], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 16:45:48.101341, \n", - " \n", - " \tupdated_at = 2024-03-05 16:45:48.101341, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", + ")\n", + "\n", + "item2 = KItem(\n", + " name=\"foo 2\",\n", + " ktype_id=dsms.ktypes.Organization,\n", + " linked_kitems=[item],\n", + " annotations=[\"www.example.org/foo\"]\n", + ")\n", + "item3 = KItem(\n", + " name=\"foo 3\", \n", + " ktype_id=dsms.ktypes.Organization\n", + ")\n", + "item4 = KItem(\n", + " name=\"foo 4\",\n", + " ktype_id=dsms.ktypes.Organization,\n", + " annotations=[\"www.example.org/bar\"],\n", + ")\n", + "\n", + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, we are apply to search for e.g. kitems of type `DatasetCatalog`:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[SearchResult(hit=KItem(\n", " \n", - " \tname = csv_bulgetest, \n", + " \tname = foo 1, \n", " \n", - " \tid = e64c512a-c2e5-428d-bc85-5d0d9c2d68c7, \n", + " \tid = 966beb55-6eb5-422e-b8c1-84d65b8cf50d, \n", " \n", " \tktype_id = dataset-catalog, \n", " \n", - " \tslug = csv_bulgetest, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=DX56_C_BLG_WR90_42.TXT), \n", - " \t\tAttachment(name=AA6014_O_BLG_WR90_241.TXT), \n", - " \t\tAttachment(name=DP800_H_BLG_WR90_140.TXT), \n", - " \t\tAttachment(name=DX56_D_BLG_WR90_56.TXT), \n", - " \t\tAttachment(name=AFZ1-BU-S8.TXT), \n", - " \t\tAttachment(name=AFZ-BU-S1.TXT), \n", - " \t\tAttachment(name=AA6014_N_BLG_WR90_218.TXT), \n", - " \t\tAttachment(name=DP800_G_BLG_WR90_126.TXT), \n", - " \t\tAttachment(name=AFZ-BU-S7.TXT), \n", - " \t\tAttachment(name=AA6014_M_BLG_WR90_195.TXT), \n", - " \t\tAttachment(name=DX56_B_BLG_WR90_28.TXT), \n", - " \t\tAttachment(name=DP800_F_BLG_WR90_108.TXT)\n", - " \t], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 16:46:41.767008, \n", - " \n", - " \tupdated_at = 2024-03-05 16:46:41.767008, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = excel_shear_tensile_test, \n", - " \n", - " \tid = e9c4bb04-9482-458d-9d12-c2ad8f214930, \n", + " \tin_backend = True, \n", " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = excel_shear_tensile_test, \n", + " \tslug = foo1-966beb55, \n", " \n", " \tannotations = [], \n", " \n", " \tattachments = [], \n", " \n", - " \tlinked_kitems = [], \n", + " \tlinked_kitems = [\n", + " \t\t\n", + " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", + " \t\t\tname: foo 2\n", + " \t\t\tslug: foo2-3ca3c293\n", + " \t\t\tktype_id: organization\n", + " \t\t\tsummary: None\n", + " \t\t\tavatar_exists: False\n", + " \t\t\tannotations: [{\n", + " \t\t\tiri: www.example.org/foo,\n", + " \t\t\tname: foo,\n", + " \t\t\tnamespace: www.example.org,\n", + " \t\t\tdescription: None\n", + " \t\t}]\n", + " \t\t\tlinked_kitems: [{\n", + " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", + " \t\t}]\n", + " \t\t\texternal_links: []\n", + " \t\t\tcontacts: []\n", + " \t\t\tauthors: [{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}]\n", + " \t\t\tlinked_affiliations: []\n", + " \t\t\tattachments: []\n", + " \t\t\tuser_groups: []\n", + " \t\t\tcustom_properties: None\n", + " \t\t\tcreated_at: 2024-08-20T08:02:09.024038\n", + " \t\t\tupdated_at: 2024-08-20T08:02:09.024038\n", + " \t\t\n", + " \t], \n", " \n", " \taffiliations = [], \n", " \n", " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", + " \t\t{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}\n", " \t], \n", " \n", " \tavatar_exists = False, \n", " \n", " \tcontacts = [], \n", " \n", - " \tcreated_at = 2024-03-05 16:47:27.228060, \n", + " \tcreated_at = 2024-08-20 08:02:08.491699, \n", " \n", - " \tupdated_at = 2024-03-05 16:47:27.228060, \n", + " \tupdated_at = 2024-08-20 08:02:08.491699, \n", " \n", " \texternal_links = [], \n", " \n", @@ -2138,188 +856,179 @@ " \n", " \tuser_groups = [], \n", " \n", - " \tcustom_properties = CustomProperties(content={}), \n", + " \tcustom_properties = None, \n", + " \n", + " \tdataframe = None, \n", " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", + " \trdf_exists = False\n", + " ), fuzzy=False)]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dsms.search(ktypes=[dsms.ktypes.DatasetCatalog])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... and for all of type `Organization` and `DatasetCatalog`:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[SearchResult(hit=KItem(\n", " \n", - " \tname = excel_tensile_test, \n", + " \tname = foo 1, \n", " \n", - " \tid = e1f1b94c-5bd6-44c7-9de4-1d01650ac7b8, \n", + " \tid = 966beb55-6eb5-422e-b8c1-84d65b8cf50d, \n", " \n", " \tktype_id = dataset-catalog, \n", " \n", - " \tslug = excel_tensile_test, \n", + " \tin_backend = True, \n", + " \n", + " \tslug = foo1-966beb55, \n", " \n", " \tannotations = [], \n", " \n", " \tattachments = [], \n", " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 16:48:01.936787, \n", - " \n", - " \tupdated_at = 2024-03-05 16:48:01.936787, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [\n", - " \t\tApp(kitem_app_id=105, executable=excel_tensile_test/excel_tensile_test.ipynb, title=excel_tensile_test, description=excel_tensile_test, tags=None, additional_properties=triggerUponUpload=True triggerUponUploadFileExtensions=['.xls', '.xlsm'])\n", - " \t], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", - " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = csv_tensile_test, \n", - " \n", - " \tid = 90207798-d6b2-47fb-b208-f7b8f771de0d, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = csv_tensile_test, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=DX56_D_FZ2_WR00_43.TXT)\n", + " \tlinked_kitems = [\n", + " \t\t\n", + " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", + " \t\t\tname: foo 2\n", + " \t\t\tslug: foo2-3ca3c293\n", + " \t\t\tktype_id: organization\n", + " \t\t\tsummary: None\n", + " \t\t\tavatar_exists: False\n", + " \t\t\tannotations: [{\n", + " \t\t\tiri: www.example.org/foo,\n", + " \t\t\tname: foo,\n", + " \t\t\tnamespace: www.example.org,\n", + " \t\t\tdescription: None\n", + " \t\t}]\n", + " \t\t\tlinked_kitems: [{\n", + " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", + " \t\t}]\n", + " \t\t\texternal_links: []\n", + " \t\t\tcontacts: []\n", + " \t\t\tauthors: [{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}]\n", + " \t\t\tlinked_affiliations: []\n", + " \t\t\tattachments: []\n", + " \t\t\tuser_groups: []\n", + " \t\t\tcustom_properties: None\n", + " \t\t\tcreated_at: 2024-08-20T08:02:09.024038\n", + " \t\t\tupdated_at: 2024-08-20T08:02:09.024038\n", + " \t\t\n", " \t], \n", " \n", - " \tlinked_kitems = [], \n", - " \n", " \taffiliations = [], \n", " \n", " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", + " \t\t{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}\n", " \t], \n", " \n", " \tavatar_exists = False, \n", " \n", " \tcontacts = [], \n", " \n", - " \tcreated_at = 2024-03-05 17:08:20.345033, \n", + " \tcreated_at = 2024-08-20 08:02:08.491699, \n", " \n", - " \tupdated_at = 2024-03-05 17:08:20.345033, \n", + " \tupdated_at = 2024-08-20 08:02:08.491699, \n", " \n", " \texternal_links = [], \n", " \n", - " \tkitem_apps = [\n", - " \t\tApp(kitem_app_id=107, executable=csv_tensile_test/csv_tensile_test.ipynb, title=csv_tensile_test, description=csv_tensile_test, tags=None, additional_properties=triggerUponUpload=True triggerUponUploadFileExtensions=['.TXT']), \n", - " \t\tApp(kitem_app_id=108, executable=csv_tensile_test/csv_tensile_test.ipynb, title=csv_tensile_test, description=csv_tensile_test, tags=None, additional_properties=triggerUponUpload=True triggerUponUploadFileExtensions=['.TXT'])\n", - " \t], \n", + " \tkitem_apps = [], \n", " \n", " \tsummary = None, \n", " \n", " \tuser_groups = [], \n", " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = [\n", - " \t\tColumn(column_id=0, name=Prüfzeit), \n", - " \t\tColumn(column_id=1, name=Standardkraft), \n", - " \t\tColumn(column_id=2, name=Traversenweg absolut), \n", - " \t\tColumn(column_id=3, name=Standardweg), \n", - " \t\tColumn(column_id=4, name=Breitenänderung), \n", - " \t\tColumn(column_id=5, name=Dehnung)\n", - " \t]\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = csv_tensile_test_f2, \n", - " \n", - " \tid = c9203684-f1ce-4d23-92f9-478a002b7440, \n", - " \n", - " \tktype_id = dataset-catalog, \n", - " \n", - " \tslug = csv_tensile_test_f2, \n", - " \n", - " \tannotations = [], \n", - " \n", - " \tattachments = [\n", - " \t\tAttachment(name=DP800_F_FZ2_WR15_97.TXT)\n", - " \t], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=83778ed4-2d3d-4fae-a1f0-35f59bb89799)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-05 17:08:27.579333, \n", - " \n", - " \tupdated_at = 2024-03-05 17:08:27.579333, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [\n", - " \t\tApp(kitem_app_id=109, executable=csv_tensile_test_f2/csv_tensile_test_f2.ipynb, title=csv_tensile_test_f2, description=csv_tensile_test_f2, tags=None, additional_properties=triggerUponUpload=True triggerUponUploadFileExtensions=['.TXT'])\n", - " \t], \n", - " \n", - " \tsummary = None, \n", + " \tcustom_properties = None, \n", " \n", - " \tuser_groups = [], \n", + " \tdataframe = None, \n", " \n", - " \tcustom_properties = CustomProperties(content={}), \n", + " \trdf_exists = False\n", + " ), fuzzy=False),\n", + " SearchResult(hit=KItem(\n", " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", + " \tname = foo 2, \n", " \n", - " \tname = material constants, \n", + " \tid = 3ca3c293-8845-4f0e-afcb-6c680c07239f, \n", " \n", - " \tid = ea0bb87b-3a26-48be-84e1-bec9b99a1204, \n", + " \tktype_id = organization, \n", " \n", - " \tktype_id = dataset-catalog, \n", + " \tin_backend = True, \n", " \n", - " \tslug = material_constants, \n", + " \tslug = foo2-3ca3c293, \n", " \n", " \tannotations = [\n", - " \t\tAnnotation(iri=https://w3id.org/steel/ProcessOntology/MaterialCard, name=MaterialCard, namespace=https://w3id.org/steel/ProcessOntology)\n", + " \t\t{\n", + " \t\t\tiri: www.example.org/foo,\n", + " \t\t\tname: foo,\n", + " \t\t\tnamespace: www.example.org,\n", + " \t\t\tdescription: None\n", + " \t\t}\n", " \t], \n", " \n", " \tattachments = [], \n", " \n", " \tlinked_kitems = [\n", - " \t\tLinkedKItem(id=72bc54cf-8138-4ac3-8586-14afe5bc6b7f, source_id=ea0bb87b-3a26-48be-84e1-bec9b99a1204), \n", - " \t\tLinkedKItem(id=253d69c1-606c-4721-832e-1caed961e8ef, source_id=ea0bb87b-3a26-48be-84e1-bec9b99a1204)\n", + " \t\t\n", + " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", + " \t\t\tname: foo 1\n", + " \t\t\tslug: foo1-966beb55\n", + " \t\t\tktype_id: dataset-catalog\n", + " \t\t\tsummary: None\n", + " \t\t\tavatar_exists: False\n", + " \t\t\tannotations: []\n", + " \t\t\tlinked_kitems: [{\n", + " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", + " \t\t}]\n", + " \t\t\texternal_links: []\n", + " \t\t\tcontacts: []\n", + " \t\t\tauthors: [{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}]\n", + " \t\t\tlinked_affiliations: []\n", + " \t\t\tattachments: []\n", + " \t\t\tuser_groups: []\n", + " \t\t\tcustom_properties: None\n", + " \t\t\tcreated_at: 2024-08-20T08:02:08.491699\n", + " \t\t\tupdated_at: 2024-08-20T08:02:08.491699\n", + " \t\t\n", " \t], \n", " \n", " \taffiliations = [], \n", " \n", " \tauthors = [\n", - " \t\tAuthor(user_id=8f40087b-16d6-49ba-a8ac-2bd70b8a2308)\n", + " \t\t{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}\n", " \t], \n", " \n", " \tavatar_exists = False, \n", " \n", " \tcontacts = [], \n", " \n", - " \tcreated_at = 2024-02-27 05:31:47.433035, \n", + " \tcreated_at = 2024-08-20 08:02:09.024038, \n", " \n", - " \tupdated_at = 2024-02-27 05:31:47.433035, \n", + " \tupdated_at = 2024-08-20 08:02:09.024038, \n", " \n", " \texternal_links = [], \n", " \n", @@ -2329,46 +1038,45 @@ " \n", " \tuser_groups = [], \n", " \n", - " \tcustom_properties = CustomProperties(content={'material constants': {'C11': 226000.0, 'C12': 140000.0, 'C44': 116000.0, 'shear_rate_ref': 0.001, 'rate_sens': 0.0125, 'tau0': 62.1930648, 'tau1': 48.1881715, 'theta0': 356.842243, 'theta1': 37.4360252, 'q1': 1.4, 'q2': 1.4}}), \n", + " \tcustom_properties = None, \n", + " \n", + " \tdataframe = None, \n", " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", + " \trdf_exists = False\n", + " ), fuzzy=False),\n", + " SearchResult(hit=KItem(\n", " \n", - " \tname = orientations, \n", + " \tname = foo 3, \n", " \n", - " \tid = 5386bf18-533a-4976-8481-d54638f07a91, \n", + " \tid = 31a58a53-8f50-4f18-93ee-90ff5a806e14, \n", " \n", - " \tktype_id = dataset-catalog, \n", + " \tktype_id = organization, \n", " \n", - " \tslug = orientations, \n", + " \tin_backend = True, \n", " \n", - " \tannotations = [\n", - " \t\tAnnotation(iri=https://w3id.org/steel/ProcessOntology/GrainOrientation, name=GrainOrientation, namespace=https://w3id.org/steel/ProcessOntology)\n", - " \t], \n", + " \tslug = foo3-31a58a53, \n", " \n", - " \tattachments = [\n", - " \t\tAttachment(name=11533_FS_Scan1_830x830_100x.ang)\n", - " \t], \n", + " \tannotations = [], \n", " \n", - " \tlinked_kitems = [\n", - " \t\tLinkedKItem(id=72bc54cf-8138-4ac3-8586-14afe5bc6b7f, source_id=5386bf18-533a-4976-8481-d54638f07a91), \n", - " \t\tLinkedKItem(id=253d69c1-606c-4721-832e-1caed961e8ef, source_id=5386bf18-533a-4976-8481-d54638f07a91)\n", - " \t], \n", + " \tattachments = [], \n", + " \n", + " \tlinked_kitems = [], \n", " \n", " \taffiliations = [], \n", " \n", " \tauthors = [\n", - " \t\tAuthor(user_id=8f40087b-16d6-49ba-a8ac-2bd70b8a2308)\n", + " \t\t{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}\n", " \t], \n", " \n", " \tavatar_exists = False, \n", " \n", " \tcontacts = [], \n", " \n", - " \tcreated_at = 2024-02-27 05:31:47.498209, \n", + " \tcreated_at = 2024-08-20 08:02:09.532080, \n", " \n", - " \tupdated_at = 2024-02-27 05:31:47.498209, \n", + " \tupdated_at = 2024-08-20 08:02:09.532080, \n", " \n", " \texternal_links = [], \n", " \n", @@ -2378,45 +1086,52 @@ " \n", " \tuser_groups = [], \n", " \n", - " \tcustom_properties = CustomProperties(content={}), \n", + " \tcustom_properties = None, \n", " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", + " \tdataframe = None, \n", " \n", - " \tname = rve_1000_grains_geofile, \n", + " \trdf_exists = False\n", + " ), fuzzy=False),\n", + " SearchResult(hit=KItem(\n", " \n", - " \tid = 13e57827-c5fe-413e-90dd-bb95ec5c1d3a, \n", + " \tname = foo 4, \n", " \n", - " \tktype_id = dataset-catalog, \n", + " \tid = 552ab1b9-64df-4343-9e4e-e5c292c3999f, \n", " \n", - " \tslug = rve_1000_grains_geofile, \n", + " \tktype_id = organization, \n", + " \n", + " \tin_backend = True, \n", + " \n", + " \tslug = foo4-552ab1b9, \n", " \n", " \tannotations = [\n", - " \t\tAnnotation(iri=https://w3id.org/steel/ProcessOntology/Geometry, name=Geometry, namespace=https://w3id.org/steel/ProcessOntology)\n", + " \t\t{\n", + " \t\t\tiri: www.example.org/bar,\n", + " \t\t\tname: bar,\n", + " \t\t\tnamespace: www.example.org,\n", + " \t\t\tdescription: None\n", + " \t\t}\n", " \t], \n", " \n", - " \tattachments = [\n", - " \t\tAttachment(name=mesh_40x40x40_grains_1000_aspect_ratio_1_6.geo)\n", - " \t], \n", + " \tattachments = [], \n", " \n", - " \tlinked_kitems = [\n", - " \t\tLinkedKItem(id=253d69c1-606c-4721-832e-1caed961e8ef, source_id=13e57827-c5fe-413e-90dd-bb95ec5c1d3a)\n", - " \t], \n", + " \tlinked_kitems = [], \n", " \n", " \taffiliations = [], \n", " \n", " \tauthors = [\n", - " \t\tAuthor(user_id=8f40087b-16d6-49ba-a8ac-2bd70b8a2308)\n", + " \t\t{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}\n", " \t], \n", " \n", " \tavatar_exists = False, \n", " \n", " \tcontacts = [], \n", " \n", - " \tcreated_at = 2024-02-27 05:31:47.588776, \n", + " \tcreated_at = 2024-08-20 08:02:10.062595, \n", " \n", - " \tupdated_at = 2024-02-27 05:31:47.588776, \n", + " \tupdated_at = 2024-08-20 08:02:10.062595, \n", " \n", " \texternal_links = [], \n", " \n", @@ -2426,19 +1141,23 @@ " \n", " \tuser_groups = [], \n", " \n", - " \tcustom_properties = CustomProperties(content={}), \n", + " \tcustom_properties = None, \n", " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", + " \tdataframe = None, \n", " \n", - " \tname = foo 3, \n", + " \trdf_exists = False\n", + " ), fuzzy=False),\n", + " SearchResult(hit=KItem(\n", + " \n", + " \tname = Research Institute ABC, \n", " \n", - " \tid = 17ec4319-7643-40e8-b3d2-a16fbafd9571, \n", + " \tid = 21aa50c3-5ec2-4ac3-aba8-69071a4287e2, \n", " \n", " \tktype_id = organization, \n", " \n", - " \tslug = foo3, \n", + " \tin_backend = True, \n", + " \n", + " \tslug = researchinstituteabc-21aa50c3, \n", " \n", " \tannotations = [], \n", " \n", @@ -2449,16 +1168,18 @@ " \taffiliations = [], \n", " \n", " \tauthors = [\n", - " \t\tAuthor(user_id=4440f8c8-f443-4114-a7a7-d8e2a6b5d776)\n", + " \t\t{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}\n", " \t], \n", " \n", " \tavatar_exists = False, \n", " \n", " \tcontacts = [], \n", " \n", - " \tcreated_at = 2024-03-06 10:13:30.805087, \n", + " \tcreated_at = 2024-08-19 18:26:00.740761, \n", " \n", - " \tupdated_at = 2024-03-06 10:13:30.805087, \n", + " \tupdated_at = 2024-08-19 18:26:00.740761, \n", " \n", " \texternal_links = [], \n", " \n", @@ -2468,13 +1189,15 @@ " \n", " \tuser_groups = [], \n", " \n", - " \tcustom_properties = CustomProperties(content={}), \n", + " \tcustom_properties = None, \n", + " \n", + " \tdataframe = None, \n", " \n", - " \thdf5 = None\n", - " )]" + " \trdf_exists = False\n", + " ), fuzzy=False)]" ] }, - "execution_count": 22, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -2492,43 +1215,74 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[KItem(\n", + "[SearchResult(hit=KItem(\n", " \n", " \tname = foo 1, \n", " \n", - " \tid = 8377f120-d558-4cc2-93d4-582bb01fac11, \n", + " \tid = 966beb55-6eb5-422e-b8c1-84d65b8cf50d, \n", " \n", " \tktype_id = dataset-catalog, \n", " \n", - " \tslug = foo1, \n", + " \tin_backend = True, \n", + " \n", + " \tslug = foo1-966beb55, \n", " \n", " \tannotations = [], \n", " \n", " \tattachments = [], \n", " \n", " \tlinked_kitems = [\n", - " \t\tLinkedKItem(id=5dacf60c-6a62-466a-a85e-9f3ce422cefa, source_id=8377f120-d558-4cc2-93d4-582bb01fac11)\n", + " \t\t\n", + " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", + " \t\t\tname: foo 2\n", + " \t\t\tslug: foo2-3ca3c293\n", + " \t\t\tktype_id: organization\n", + " \t\t\tsummary: None\n", + " \t\t\tavatar_exists: False\n", + " \t\t\tannotations: [{\n", + " \t\t\tiri: www.example.org/foo,\n", + " \t\t\tname: foo,\n", + " \t\t\tnamespace: www.example.org,\n", + " \t\t\tdescription: None\n", + " \t\t}]\n", + " \t\t\tlinked_kitems: [{\n", + " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", + " \t\t}]\n", + " \t\t\texternal_links: []\n", + " \t\t\tcontacts: []\n", + " \t\t\tauthors: [{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}]\n", + " \t\t\tlinked_affiliations: []\n", + " \t\t\tattachments: []\n", + " \t\t\tuser_groups: []\n", + " \t\t\tcustom_properties: None\n", + " \t\t\tcreated_at: 2024-08-20T08:02:09.024038\n", + " \t\t\tupdated_at: 2024-08-20T08:02:09.024038\n", + " \t\t\n", " \t], \n", " \n", " \taffiliations = [], \n", " \n", " \tauthors = [\n", - " \t\tAuthor(user_id=4440f8c8-f443-4114-a7a7-d8e2a6b5d776)\n", + " \t\t{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}\n", " \t], \n", " \n", " \tavatar_exists = False, \n", " \n", " \tcontacts = [], \n", " \n", - " \tcreated_at = 2024-03-06 10:13:29.213578, \n", + " \tcreated_at = 2024-08-20 08:02:08.491699, \n", " \n", - " \tupdated_at = 2024-03-06 10:13:29.213578, \n", + " \tupdated_at = 2024-08-20 08:02:08.491699, \n", " \n", " \texternal_links = [], \n", " \n", @@ -2538,13 +1292,15 @@ " \n", " \tuser_groups = [], \n", " \n", - " \tcustom_properties = CustomProperties(content={}), \n", + " \tcustom_properties = None, \n", " \n", - " \thdf5 = None\n", - " )]" + " \tdataframe = None, \n", + " \n", + " \trdf_exists = False\n", + " ), fuzzy=False)]" ] }, - "execution_count": 23, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -2562,45 +1318,76 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[KItem(\n", + "[SearchResult(hit=KItem(\n", " \n", " \tname = foo 2, \n", " \n", - " \tid = 5dacf60c-6a62-466a-a85e-9f3ce422cefa, \n", + " \tid = 3ca3c293-8845-4f0e-afcb-6c680c07239f, \n", " \n", " \tktype_id = organization, \n", " \n", - " \tslug = foo2, \n", + " \tin_backend = True, \n", + " \n", + " \tslug = foo2-3ca3c293, \n", " \n", " \tannotations = [\n", - " \t\tAnnotation(iri=www.example.org/foo, name=foo, namespace=www.example.org)\n", + " \t\t{\n", + " \t\t\tiri: www.example.org/foo,\n", + " \t\t\tname: foo,\n", + " \t\t\tnamespace: www.example.org,\n", + " \t\t\tdescription: None\n", + " \t\t}\n", " \t], \n", " \n", " \tattachments = [], \n", " \n", " \tlinked_kitems = [\n", - " \t\tLinkedKItem(id=8377f120-d558-4cc2-93d4-582bb01fac11, source_id=5dacf60c-6a62-466a-a85e-9f3ce422cefa)\n", + " \t\t\n", + " \t\t\tid: 966beb55-6eb5-422e-b8c1-84d65b8cf50d\n", + " \t\t\tname: foo 1\n", + " \t\t\tslug: foo1-966beb55\n", + " \t\t\tktype_id: dataset-catalog\n", + " \t\t\tsummary: None\n", + " \t\t\tavatar_exists: False\n", + " \t\t\tannotations: []\n", + " \t\t\tlinked_kitems: [{\n", + " \t\t\tid: 3ca3c293-8845-4f0e-afcb-6c680c07239f\n", + " \t\t}]\n", + " \t\t\texternal_links: []\n", + " \t\t\tcontacts: []\n", + " \t\t\tauthors: [{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}]\n", + " \t\t\tlinked_affiliations: []\n", + " \t\t\tattachments: []\n", + " \t\t\tuser_groups: []\n", + " \t\t\tcustom_properties: None\n", + " \t\t\tcreated_at: 2024-08-20T08:02:08.491699\n", + " \t\t\tupdated_at: 2024-08-20T08:02:08.491699\n", + " \t\t\n", " \t], \n", " \n", " \taffiliations = [], \n", " \n", " \tauthors = [\n", - " \t\tAuthor(user_id=4440f8c8-f443-4114-a7a7-d8e2a6b5d776)\n", + " \t\t{\n", + " \t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + " \t\t}\n", " \t], \n", " \n", " \tavatar_exists = False, \n", " \n", " \tcontacts = [], \n", " \n", - " \tcreated_at = 2024-03-06 10:13:29.996514, \n", + " \tcreated_at = 2024-08-20 08:02:09.024038, \n", " \n", - " \tupdated_at = 2024-03-06 10:13:29.996514, \n", + " \tupdated_at = 2024-08-20 08:02:09.024038, \n", " \n", " \texternal_links = [], \n", " \n", @@ -2610,57 +1397,15 @@ " \n", " \tuser_groups = [], \n", " \n", - " \tcustom_properties = CustomProperties(content={}), \n", - " \n", - " \thdf5 = None\n", - " ),\n", - " KItem(\n", - " \n", - " \tname = foo 4, \n", - " \n", - " \tid = 6e0574de-e358-4305-9148-244132506044, \n", - " \n", - " \tktype_id = organization, \n", - " \n", - " \tslug = foo4, \n", - " \n", - " \tannotations = [\n", - " \t\tAnnotation(iri=www.example.org/bar, name=bar, namespace=https://www.example.org)\n", - " \t], \n", - " \n", - " \tattachments = [], \n", - " \n", - " \tlinked_kitems = [], \n", - " \n", - " \taffiliations = [], \n", - " \n", - " \tauthors = [\n", - " \t\tAuthor(user_id=4440f8c8-f443-4114-a7a7-d8e2a6b5d776)\n", - " \t], \n", - " \n", - " \tavatar_exists = False, \n", - " \n", - " \tcontacts = [], \n", - " \n", - " \tcreated_at = 2024-03-06 10:13:31.643039, \n", - " \n", - " \tupdated_at = 2024-03-06 10:13:31.643039, \n", - " \n", - " \texternal_links = [], \n", - " \n", - " \tkitem_apps = [], \n", - " \n", - " \tsummary = None, \n", - " \n", - " \tuser_groups = [], \n", + " \tcustom_properties = None, \n", " \n", - " \tcustom_properties = CustomProperties(content={}), \n", + " \tdataframe = None, \n", " \n", - " \thdf5 = None\n", - " )]" + " \trdf_exists = False\n", + " ), fuzzy=False)]" ] }, - "execution_count": 24, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -2680,29 +1425,16 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 22, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "del dsms[item]\n", "del dsms[item2]\n", "del dsms[item3]\n", "del dsms[item4]\n", "\n", - "dsms.commit()\n", - "\n", - "dsms.kitems" + "dsms.commit()\n" ] }, { @@ -2721,27 +1453,28 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[App(filename='bulk_clone_pipelines.ipynb', basename='bulk_clone_pipelines.ipynb', folder=''),\n", - " App(filename='csv_tensile_test/csv_tensile_test.ipynb', basename='csv_tensile_test.ipynb', folder='csv_tensile_test'),\n", - " App(filename='csv_tensile_test_f2/csv_tensile_test_f2.ipynb', basename='csv_tensile_test_f2.ipynb', folder='csv_tensile_test_f2'),\n", - " App(filename='excel_nakajima_test/excel_nakajima_test.ipynb', basename='excel_nakajima_test.ipynb', folder='excel_nakajima_test'),\n", - " App(filename='excel_shear_test/excel_shear_tensile_test.ipynb', basename='excel_shear_tensile_test.ipynb', folder='excel_shear_test'),\n", - " App(filename='excel_tensile_test/excel_tensile_test.ipynb', basename='excel_tensile_test.ipynb', folder='excel_tensile_test')]" + "[AppConfig(name=ckan-fetch, specification={'metadata': {'generateName': 'ckan-resource-request-'}}),\n", + " AppConfig(name=csv_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=csv_tensile_test_f2, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=excel_notched_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=excel_shear_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=excel_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=ternary-plot, specification={'metadata': {'generateName': 'ckan-tenary-app-'}})]" ] }, - "execution_count": 26, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dsms.apps" + "dsms.app_configs" ] }, { @@ -2760,7 +1493,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -2769,23 +1502,23 @@ "text": [ "Column-wise:\n", "column: a ,\n", - " data: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 50.0, 51.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0, 75.0, 76.0, 77.0, 78.0, 79.0, 80.0, 81.0, 82.0, 83.0, 84.0, 85.0, 86.0, 87.0, 88.0, 89.0, 90.0, 91.0, 92.0, 93.0, 94.0, 95.0, 96.0, 97.0, 98.0, 99.0]\n", + " data: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]\n", "column: b ,\n", - " data: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 50.0, 51.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0, 75.0, 76.0, 77.0, 78.0, 79.0, 80.0, 81.0, 82.0, 83.0, 84.0, 85.0, 86.0, 87.0, 88.0, 89.0, 90.0, 91.0, 92.0, 93.0, 94.0, 95.0, 96.0, 97.0, 98.0, 99.0, 100.0]\n", + " data: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100]\n", "\n", "As data frame:\n", - " a b\n", - "0 0.0 1.0\n", - "1 1.0 2.0\n", - "2 2.0 3.0\n", - "3 3.0 4.0\n", - "4 4.0 5.0\n", - ".. ... ...\n", - "95 95.0 96.0\n", - "96 96.0 97.0\n", - "97 97.0 98.0\n", - "98 98.0 99.0\n", - "99 99.0 100.0\n", + " a b\n", + "0 0 1\n", + "1 1 2\n", + "2 2 3\n", + "3 3 4\n", + "4 4 5\n", + ".. .. ...\n", + "95 95 96\n", + "96 96 97\n", + "97 97 98\n", + "98 98 99\n", + "99 99 100\n", "\n", "[100 rows x 2 columns]\n" ] @@ -2795,26 +1528,26 @@ "data = {\"a\": list(range(100)), \"b\": list(range(1,101))}\n", "\n", "\n", - "item = KItem(name=\"testdata123\", ktype_id=dsms.ktypes.DatasetCatalog, hdf5=data)\n", + "item = KItem(name=\"testdata1234\", ktype_id=dsms.ktypes.DatasetCatalog, dataframe=data)\n", "dsms.commit()\n", "\n", "print(\"Column-wise:\")\n", - "for column in item.hdf5:\n", + "for column in item.dataframe:\n", " print(\"column:\", column.name, \",\\n\", \"data:\", column.get())\n", "\n", - "df = item.hdf5.to_df()\n", + "df = item.dataframe.to_df()\n", "print(\"\\nAs data frame:\")\n", "print(df)\n", "\n", "new_df = df.drop(['a'], axis=1)\n", - "item.hdf5 = new_df\n", + "item.dataframe = new_df\n", "\n", "dsms.commit()" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -2823,7 +1556,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -2847,7 +1580,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.1" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/examples/tutorials/1_introduction.ipynb b/examples/tutorials/1_introduction.ipynb new file mode 100644 index 0000000..a727932 --- /dev/null +++ b/examples/tutorials/1_introduction.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 1. Connecting with the SDK to DSMS\n", + "\n", + "In this tutorial we see the overview on how to setup and basic use DSMS-SDK\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.1. Setting up" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2. Introduction to KItems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see which kind of DSMS-object we own as a user (in the beginning, we own none):" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dsms.kitems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can investigate what a KItem needs in order to be created. KItems are entirely based on Pydantic-Models (v2), hence the properties (in Pydantic called Fields) are automatically validated once we set them.\n", + "\n", + "The schema of the KItem itself is a JSON schema which is machine-readable and can be directly incorporated into Swagger-supported APIs like e.g. FastAPI.\n", + "\n", + "We can investigate the KTypes defined in the remote instance:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KTypes.Organization\n", + "KTypes.App\n", + "KTypes.Dataset\n", + "KTypes.DatasetCatalog\n", + "KTypes.Expert\n", + "KTypes.Test\n", + "KTypes.Specimen\n", + "KTypes.Batch\n", + "KTypes.Resource\n", + "KTypes.TestingMachine\n" + ] + } + ], + "source": [ + "for ktype in dsms.ktypes:\n", + " print(ktype)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/2_creation.ipynb b/examples/tutorials/2_creation.ipynb new file mode 100644 index 0000000..725bb64 --- /dev/null +++ b/examples/tutorials/2_creation.ipynb @@ -0,0 +1,330 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 2. Create KItems with the SDK\n", + "\n", + "In this tutorial we see how to create new Kitems." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1. Setting up\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KItem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "### 2.2: Create KItems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can make new KItems by simple class-initiation: (Make sure existing KItems are not given as input). \n", + "#" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = KTypes.TestingMachine, \n", + "\n", + "\tin_backend = False, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = None, \n", + "\n", + "\tupdated_at = None, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: TestingLab GmBH, \n", + "\t\tLocation: A404, \n", + "\t\tModel Number: Bending Test Machine No 777\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item = KItem(\n", + " name=\"Machine-1\",\n", + " ktype_id=dsms.ktypes.TestingMachine,\n", + " custom_properties={\"Producer\": \"TestingLab GmBH\",\n", + " \"Location\": \"A404\",\n", + " \"Model Number\" : \"Bending Test Machine No 777\"\n", + " },\n", + ")\n", + "\n", + "item" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Remember: changes are only syncronized with the DSMS when you call the `commit`-method:" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'https://bue.materials-data.space/knowledge/testing-machine/machine-1-dd091666'" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dsms.commit()\n", + "item.url" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, the object we created before running the `commit`-method has automatically been updated, e.g. with the creation- and update-timestamp. We can check this with the below command:" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\tupdated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: TestingLab GmBH, \n", + "\t\tLocation: A404, \n", + "\t\tModel Number: Bending Test Machine No 777\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To just get the name of the item, we can do it as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Machine-1'" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item.name" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To check the type of the item newly created we can use the following:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dsms.knowledge.kitem.KItem" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "type(item)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Now you can check if the particular kitem is in the list of KItems. This can be done either by using the command:\n", + " `\n", + " dsms.kitems\n", + " `\n", + " or by logging into the frontend dsms instance." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/3_updation.ipynb b/examples/tutorials/3_updation.ipynb new file mode 100644 index 0000000..3bf4d84 --- /dev/null +++ b/examples/tutorials/3_updation.ipynb @@ -0,0 +1,287 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 3. Updating KItems with the SDK\n", + "\n", + "In this tutorial we see how to update existing Kitems." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.1. Setting up\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now lets get the kitem we created in the [2nd tutorial : Creation of Kitems](2_creation.ipynb)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Machine-1\n" + ] + } + ], + "source": [ + "item = dsms.kitems[-1]\n", + "print(item.name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.2. Updating Kitems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, we would like to update the properties of our KItem we created previously.\n", + "\n", + "Depending on the schema of each property (see [DSMS KItem Schema](../dsms_kitem_schema.md)), we can simply use the standard `list`-method as we know them from basic Python (e.g. for the `annotations`, `attachments`, `external_link`, etc). \n", + "\n", + "Other properties which are not `list`-like can be simply set by attribute-assignment (e.g. `name`, `slug`, `ktype_id`, etc)." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "item.name = \"Machine-1\"\n", + "item.custom_properties.Producer = \"Machinery GmBH\"\n", + "item.attachments.append(\"testfile.txt\")\n", + "item.annotations.append(\"www.machinery.org/\")\n", + "item.external_links.append(\n", + " {\"url\": \"http://machine.org\", \"label\": \"machine-link\"}\n", + ")\n", + "item.contacts.append({\"name\": \"machinesupport\", \"email\": \"machinesupport@group.mail\"})\n", + "item.affiliations.append(\"machine-team\")\n", + "item.user_groups.append({\"name\": \"machinegroup\", \"group_id\": \"123\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see now that the local system path of the attachment is changed to a simply file name, which means that the upload was successful. If not so, an error would have been thrown during the `commit`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see the updates when we print the item:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [\n", + "\t\t{\n", + "\t\t\tiri: www.machinery.org/,\n", + "\t\t\tname: ,\n", + "\t\t\tnamespace: www.machinery.org,\n", + "\t\t\tdescription: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tattachments = [\n", + "\t\t{\n", + "\t\t\tname: testfile.txt\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [\n", + "\t\t{\n", + "\t\t\tname: machine-team\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [\n", + "\t\t{\n", + "\t\t\tname: machinesupport,\n", + "\t\t\temail: machinesupport@group.mail,\n", + "\t\t\tuser_id: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcreated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\tupdated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\texternal_links = [\n", + "\t\t{\n", + "\t\t\tlabel: machine-link,\n", + "\t\t\turl: http://machine.org/\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [\n", + "\t\t{\n", + "\t\t\tname: machinegroup,\n", + "\t\t\tgroup_id: 123\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: Machinery GmBH, \n", + "\t\tLocation: A404, \n", + "\t\tModel Number: Bending Test Machine No 777\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Furthermore we can also download the file we uploaded again:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\t\t\t Downloaded file: {\n", + "\t\t\tname: testfile.txt\n", + "\t\t}\n", + "|------------------------------------Beginning of file------------------------------------|\n", + "This is a calibration protocol!\n", + "|---------------------------------------End of file---------------------------------------|\n" + ] + } + ], + "source": [ + "for file in item.attachments:\n", + " download = file.download()\n", + "\n", + " print(\"\\t\\t\\t Downloaded file:\", file)\n", + " print(\"|------------------------------------Beginning of file------------------------------------|\")\n", + " print(download)\n", + " print(\"|---------------------------------------End of file---------------------------------------|\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/4_deletion.ipynb b/examples/tutorials/4_deletion.ipynb new file mode 100644 index 0000000..210df6b --- /dev/null +++ b/examples/tutorials/4_deletion.ipynb @@ -0,0 +1,404 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 4. Deleting KItems with the SDK\n", + "\n", + "In this tutorial we see how to delete new Kitems and their properties." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.1. Setting up\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then lets see the Kitem we are interested in to remove." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [\n", + "\t\t{\n", + "\t\t\tiri: www.machinery.org/,\n", + "\t\t\tname: ,\n", + "\t\t\tnamespace: www.machinery.org,\n", + "\t\t\tdescription: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tattachments = [\n", + "\t\t{\n", + "\t\t\tname: testfile.txt\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [\n", + "\t\t{\n", + "\t\t\tname: machine-team\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [\n", + "\t\t{\n", + "\t\t\tname: machinesupport,\n", + "\t\t\temail: machinesupport@group.mail,\n", + "\t\t\tuser_id: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcreated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\tupdated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\texternal_links = [\n", + "\t\t{\n", + "\t\t\tlabel: machine-link,\n", + "\t\t\turl: http://machine.org/\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [\n", + "\t\t{\n", + "\t\t\tname: machinegroup,\n", + "\t\t\tgroup_id: 123\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: Machinery GmBH, \n", + "\t\tLocation: A404, \n", + "\t\tModel Number: Bending Test Machine No 777\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")\n" + ] + } + ], + "source": [ + "item = dsms.kitems[-1]\n", + "print(item)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.2. Deletion of KItems and their properties" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also remove properties from the KItem without deleting the KItem itself.\n", + "\n", + "For the `list`-like properties, we can use the standard `list`-methods from basic Python again (e.g. `pop`, `remove`, etc. or the `del`-operator).\n", + "\n", + "For the other, non-`list`-like properties, we can simply use the attribute-assignment again.\n", + "\n", + "When we only want single parts of the properties in the KItem, we can do it like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{\n", + "\t\t\tname: machinegroup,\n", + "\t\t\tgroup_id: 123\n", + "\t\t}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item.attachments.pop(0)\n", + "item.annotations.pop(0)\n", + "item.external_links.pop(0)\n", + "item.contacts.pop(0)\n", + "item.user_groups.pop(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, we can also reset the entire property by setting it to e.g. an empty list again:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "item.affiliations = []" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can delete the custom properties by setting the property to an empty dict:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "item.custom_properties = {}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Send the changes to the DSMS with the `commit`-method:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "See the changes:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-dd091666, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [\n", + "\t\t{\n", + "\t\t\tname: machine-team\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [\n", + "\t\t{\n", + "\t\t\tname: machinesupport,\n", + "\t\t\temail: machinesupport@group.mail,\n", + "\t\t\tuser_id: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tcreated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\tupdated_at = 2024-08-19 18:12:11.338394, \n", + "\n", + "\texternal_links = [\n", + "\t\t{\n", + "\t\t\tlabel: machine-link,\n", + "\t\t\turl: http://machine.org/\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tid: dd091666-a7c9-4b3b-8832-910bdec5c63c, \n", + "\t\tcontent: {}\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, we can also delete the whole KItem from the DSMS by applying the `del`-operator to the `dsms`-object with the individual `KItem`-object:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "del dsms[item]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Commit the changes:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now to check if the particular kitem was removed, we can do this by using the command:\n", + " `\n", + " dsms.kitems\n", + " `\n", + " or by logging into the frontend dsms instance." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/5_search.ipynb b/examples/tutorials/5_search.ipynb new file mode 100644 index 0000000..eae274a --- /dev/null +++ b/examples/tutorials/5_search.ipynb @@ -0,0 +1,481 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 5. Searching KItems with the SDK\n", + "\n", + "In this tutorial we see how to search existing Kitems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.1. Setting up\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KItem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.2. Searching for KItems" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this section, we would like to search for specfic KItems we created in the DSMS.\n", + "\n", + "For this purpose, we will firstly create some KItems and apply the `search`-method on the `DSMS`-object later on in order to find them again in the DSMS.\n", + "\n", + "We also want to demonstrate here, that we can link KItems to each other in order to find e.g. a related item of type `DatasetCatalog`. For this strategy, we are using the `linked_kitems`- attribute and the `id` of the item which we would like to link.\n", + "\n", + "The procedure looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "item1 = KItem(\n", + " name=\"Machine-1\",\n", + " ktype_id=dsms.ktypes.TestingMachine,\n", + " custom_properties={\"Producer\": \"TestingLab GmBH\",\n", + " \"Room Number\": \"A404\",\n", + " \"Description\": \"Bending Test Machine\"\n", + " }\n", + ")\n", + "\n", + "item2 = KItem(\n", + " name=\"Machine-2\",\n", + " ktype_id=dsms.ktypes.TestingMachine,\n", + " custom_properties={\"Producer\": \"StressStrain GmBH\",\n", + " \"Room Number\": \"B500\",\n", + " \"Description\": \"Compression Test Machine\"\n", + " }\n", + ")\n", + "\n", + "item3 = KItem(\n", + " name=\"Specimen-1\", \n", + " ktype_id=dsms.ktypes.Specimen,\n", + " linked_kitems=[item1],\n", + " custom_properties={\"Geometry\": \"Cylindrical 150mm x 20mm x 40mm\",\n", + " \"Material\": \"Concrete\",\n", + " \"Project ID\": \"ConstructionProject2024\"\n", + " }\n", + "\n", + ")\n", + "item4 = KItem(\n", + " name=\"Specimen-2\",\n", + " ktype_id=dsms.ktypes.Specimen,\n", + " linked_kitems=[item2],\n", + " custom_properties={\"Geometry\": \"Rectangular 200mm x 30mm x 20mm\",\n", + " \"Material\": \"Metal\",\n", + " \"Project ID\": \"MetalBlenders2024\"\n", + " }\n", + ")\n", + "\n", + "item5 = KItem(\n", + " name=\"Research Institute ABC\",\n", + " ktype_id=dsms.ktypes.Organization,\n", + " linked_kitems=[item1],\n", + " annotations=[\n", + " {\n", + " \"iri\": \"www.researchBACiri.org/foo\",\n", + " \"name\": \"research ABC Institute\",\n", + " \"namespace\": \"research\",\n", + " }\n", + " ],\n", + ")\n", + "\n", + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "####

Note : Here in this tutorial, we use dsms.search with `limit=1` to maintain readability but the user can adjust the variable `limit` as per requirement.

\n", + "\n", + "\n", + "Now, we are apply to search for e.g. kitems of type `TestingMachine`:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = Machine-1, \n", + "\n", + "\tid = 0f49b0ad-2a98-4c10-8440-fd3c37363646, \n", + "\n", + "\tktype_id = testing-machine, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = machine-1-0f49b0ad, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [\n", + "\t\t\n", + "\t\t\tid: 075709da-5481-4cb4-a8ee-c18db34ce9d3\n", + "\t\t\tname: Specimen-1\n", + "\t\t\tslug: specimen-1-075709da\n", + "\t\t\tktype_id: specimen\n", + "\t\t\tsummary: None\n", + "\t\t\tavatar_exists: False\n", + "\t\t\tannotations: []\n", + "\t\t\tlinked_kitems: [{\n", + "\t\t\tid: 0f49b0ad-2a98-4c10-8440-fd3c37363646\n", + "\t\t}]\n", + "\t\t\texternal_links: []\n", + "\t\t\tcontacts: []\n", + "\t\t\tauthors: [{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}]\n", + "\t\t\tlinked_affiliations: []\n", + "\t\t\tattachments: []\n", + "\t\t\tuser_groups: []\n", + "\t\t\tcustom_properties: {'sampletype': None, 'sampleinformation': None, 'sampleproductionprocess': None, 'Geometry': 'Cylindrical 150mm x 20mm x 40mm', 'Material': 'Concrete', 'Project ID': 'ConstructionProject2024'}\n", + "\t\t\tcreated_at: 2024-08-19T18:26:00.499708\n", + "\t\t\tupdated_at: 2024-08-19T18:26:00.499708\n", + "\t\t\n", + "\t], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-19 18:26:00.247787, \n", + "\n", + "\tupdated_at = 2024-08-19 18:26:00.247787, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = {\n", + "\t\tProducer: TestingLab GmBH, \n", + "\t\tRoom Number: A404, \n", + "\t\tDescription: Bending Test Machine\n", + "\t}, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")\n", + "\n", + "\n" + ] + } + ], + "source": [ + "for result in dsms.search(ktypes=[dsms.ktypes.TestingMachine], limit=1):\n", + " print(result.hit)\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... and for all of type `Organization` and `DatasetCatalog`:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = Research Institute ABC, \n", + "\n", + "\tid = 21aa50c3-5ec2-4ac3-aba8-69071a4287e2, \n", + "\n", + "\tktype_id = organization, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = researchinstituteabc-21aa50c3, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-19 18:26:00.740761, \n", + "\n", + "\tupdated_at = 2024-08-19 18:26:00.740761, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = None, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")\n", + "fuzziness: False\n", + "\n", + "\n" + ] + } + ], + "source": [ + "for result in dsms.search(ktypes=[dsms.ktypes.Organization, dsms.ktypes.DatasetCatalog], limit=1):\n", + " print(result.hit)\n", + " print(\"fuzziness: \", result.fuzzy)\n", + " print(\"\\n\")\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... or for all of type `Dataset` with `Specimen-1` in the name:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "for result in dsms.search(query=\"Specimen-1\", ktypes=[dsms.ktypes.Dataset], limit=1):\n", + " print(result.hit)\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... and for all of type `Organization` with the annotation `www.researchBACiri.org/foo`:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = Research Institute ABC, \n", + "\n", + "\tid = 84c089e6-feab-4ba6-a934-527594e504eb, \n", + "\n", + "\tktype_id = organization, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = researchinstituteabc-84c089e6, \n", + "\n", + "\tannotations = [\n", + "\t\t{\n", + "\t\t\tiri: www.researchBACiri.org/foo,\n", + "\t\t\tname: research ABC Institute,\n", + "\t\t\tnamespace: research,\n", + "\t\t\tdescription: None\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tattachments = [], \n", + "\n", + "\tlinked_kitems = [\n", + "\t\t\n", + "\t\t\tid: 694561e9-df15-4c71-ae71-abd91df3ec23\n", + "\t\t\tname: Machine-1\n", + "\t\t\tslug: machine-1-694561e9\n", + "\t\t\tktype_id: testing-machine\n", + "\t\t\tsummary: None\n", + "\t\t\tavatar_exists: False\n", + "\t\t\tannotations: []\n", + "\t\t\tlinked_kitems: [{\n", + "\t\t\tid: 83672cdb-7584-4dd7-9b6e-d10574f1adf7\n", + "\t\t}, {\n", + "\t\t\tid: 84c089e6-feab-4ba6-a934-527594e504eb\n", + "\t\t}]\n", + "\t\t\texternal_links: []\n", + "\t\t\tcontacts: []\n", + "\t\t\tauthors: [{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}]\n", + "\t\t\tlinked_affiliations: []\n", + "\t\t\tattachments: []\n", + "\t\t\tuser_groups: []\n", + "\t\t\tcustom_properties: {'Producer': 'TestingLab GmBH', 'Room Number': 'A404', 'Description': 'Bending Test Machine'}\n", + "\t\t\tcreated_at: 2024-08-19T18:26:08.898435\n", + "\t\t\tupdated_at: 2024-08-19T18:26:08.898435\n", + "\t\t\n", + "\t], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = False, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-19 18:26:09.390800, \n", + "\n", + "\tupdated_at = 2024-08-19 18:26:09.390800, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = None, \n", + "\n", + "\tdataframe = None, \n", + "\n", + "\trdf_exists = False\n", + ")\n", + "\n", + "\n" + ] + } + ], + "source": [ + "for result in dsms.search(\n", + " ktypes=[dsms.ktypes.Organization], annotations=[\"www.researchBACiri.org/foo\"], limit=1\n", + " ):\n", + " print(result.hit)\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Clean up the DSMS from the tutortial:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "del dsms[item1]\n", + "del dsms[item2]\n", + "del dsms[item3]\n", + "del dsms[item4]\n", + "del dsms[item5]\n", + "\n", + "dsms.commit()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/6_apps.ipynb b/examples/tutorials/6_apps.ipynb new file mode 100644 index 0000000..48af684 --- /dev/null +++ b/examples/tutorials/6_apps.ipynb @@ -0,0 +1,1025 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 6. DSMS Apps and Pipelines\n", + "\n", + "In this tutorial we see how to create apps and run them manually" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.1: Setting up\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from dsms import DSMS, KItem, AppConfig\n", + "import time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now source the environmental variables from an `.env` file and start the DSMS-session." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "dsms = DSMS(env=\"../../.env\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.1. Investigating Available Apps" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can investigate which apps are available:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[AppConfig(name=ckan-fetch, specification={'metadata': {'generateName': 'ckan-resource-request-'}}),\n", + " AppConfig(name=csv_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=csv_tensile_test_f2, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=excel_notched_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=excel_shear_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=excel_tensile_test, specification={'metadata': {'generateName': 'data2rdf-'}}),\n", + " AppConfig(name=ternary-plot, specification={'metadata': {'generateName': 'ckan-tenary-app-'}})]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dsms.app_configs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.2 Create a new app config and apply it to a KItem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.2.1 Arbitrary python code" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To be defined." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.2.2 - Data2RDF" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 6.2.2.1 Prepare app and its config" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the following example, we would like to upload some csv with some arbitrary data and describe it through an RDF. This will give us the opportunity to harmonize the entities of the data file through ontological concepts and allow us to convert values of the data frame columns in to any compatible unit we would like to have.\n", + "\n", + "Fist of all, let us define the data:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "data = \"\"\"A,B,C\n", + "1.2,1.3,1.5\n", + "1.7,1.8,1.9\n", + "2.0,2.1,2.3\n", + "2.5,2.6,2.8\n", + "3.0,3.2,3.4\n", + "3.6,3.7,3.9\n", + "4.1,4.3,4.4\n", + "4.7,4.8,5.0\n", + "5.2,5.3,5.5\n", + "5.8,6.0,6.1\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will also give the config a defined name:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "configname = \"testapp2\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As a next step, we want to create a new app specification. The specification is following the definition of an [**Argo Workflow**](https://argo-workflows.readthedocs.io/en/latest/). The workflow shall trigger a pipeline from a docker image with the [**Data2RDF package**](https://data2rdf.readthedocs.io/en/latest/). \n", + "\n", + "The image has already been deployed on the k8s cluster of the DSMS and the workflow template with the name `dsms-data2rdf` has been implemented previously. Hence we only need to configure our pipeline for our data shown above, which we would like to upload and describe through an RDF.\n", + "\n", + "For more details about the data2rdf package, please refer to the documentation of Data2RDF mentioned above.\n", + "\n", + "The parameters of the app config are defining the inputs for our Data2RDF pipeline. This e.g. are: \n", + "\n", + "* the parser kind (`csv` here)\n", + "* the time series header length (`1` here)\n", + "* the metadata length (`0` here)\n", + "* the time series separator (`,` here)\n", + "* the log level (`DEBUG` here)\n", + "* the mapping \n", + " * `A` is the test time and has a unit in seconds\n", + " * `B` is the standard force in kilonewtons\n", + " * `C` is the absolut cross head travel in millimeters" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "parameters = [\n", + " {\"name\": \"parser\", \"value\": \"csv\"},\n", + " {\"name\": \"time_series_header_length\", \"value\": 1},\n", + " {\"name\": \"metadata_length\", \"value\": 0},\n", + " {\"name\": \"time_series_sep\", \"value\": \",\"},\n", + " {\"name\": \"log_level\", \"value\": \"DEBUG\"},\n", + " {\n", + " \"name\": \"mapping\",\n", + " \"value\": \"\"\"\n", + " [\n", + " {\n", + " \"key\": \"A\",\n", + " \"iri\": \"https://w3id.org/steel/ProcessOntology/TestTime\",\n", + " \"unit\": \"s\"\n", + " },\n", + " {\n", + " \"key\": \"B\",\n", + " \"iri\": \"https://w3id.org/steel/ProcessOntology/StandardForce\",\n", + " \"unit\": \"kN\"\n", + " },\n", + " {\n", + " \"key\": \"C\",\n", + " \"iri\": \"https://w3id.org/steel/ProcessOntology/AbsoluteCrossheadTravel\",\n", + " \"unit\": \"mm\"\n", + " }\n", + " ]\n", + " \"\"\",\n", + " },\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we add the parameters to our app specification. We assign a prefix `datardf-` which shall generate a new name with some random characters as suffix. The workflow template with the Docker image we want to run is called `dsms-data2rdf` and its `entrypoint` is `execute_pipeline`." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Define app specification\n", + "specification = {\n", + " \"apiVersion\": \"argoproj.io/v1alpha1\",\n", + " \"kind\": \"Workflow\",\n", + " \"metadata\": {\"generateName\": \"data2rdf-\"},\n", + " \"spec\": {\n", + " \"entrypoint\": \"execute_pipeline\",\n", + " \"workflowTemplateRef\": {\"name\": \"dsms-data2rdf\"},\n", + " \"arguments\": {\"parameters\": parameters},\n", + " },\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we instanciate the new app config:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "appspec = AppConfig(\n", + " name=configname,\n", + " specification=specification, # this can also be a file path to a yaml file instead of a dict\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We commit the new app config:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we would like to apply the app config to a KItem. The set the `triggerUponUpload` must be set to `True` so that the app is triggered automatically when we upload an attachment.\n", + "\n", + "Additionally, we must tell the file extension for which the upload shall be triggered. Here it is `.csv`.\n", + "\n", + "We also want to generate a qr code as avatar for the KItem with `avatar={\"include_qr\": True}`." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "item = KItem(\n", + " name=\"my tensile test experiment\",\n", + " ktype_id=dsms.ktypes.Dataset,\n", + " kitem_apps=[\n", + " {\n", + " \"executable\": appspec.name,\n", + " \"title\": \"data2rdf\",\n", + " \"additional_properties\": {\n", + " \"triggerUponUpload\": True,\n", + " \"triggerUponUploadFileExtensions\": [\".csv\"],\n", + " },\n", + " }\n", + " ],\n", + " avatar={\"include_qr\": True},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We commit the KItem:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we add our data with our attachment:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "item.attachments = [{\"name\": \"dummy_data.csv\", \"content\": data}]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And we commit again:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 6.2.2.2 Get results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can verify that the data extraction was successful:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "KItem(\n", + "\n", + "\tname = my tensile test experiment, \n", + "\n", + "\tid = 58683a2d-7823-4638-bc8e-91b461afa593, \n", + "\n", + "\tktype_id = dataset, \n", + "\n", + "\tin_backend = True, \n", + "\n", + "\tslug = mytensiletestexperiment-58683a2d, \n", + "\n", + "\tannotations = [], \n", + "\n", + "\tattachments = [\n", + "\t\t{\n", + "\t\t\tname: dummy_data.csv\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tlinked_kitems = [], \n", + "\n", + "\taffiliations = [], \n", + "\n", + "\tauthors = [\n", + "\t\t{\n", + "\t\t\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tavatar_exists = True, \n", + "\n", + "\tcontacts = [], \n", + "\n", + "\tcreated_at = 2024-08-20 08:04:25.756982, \n", + "\n", + "\tupdated_at = 2024-08-20 08:04:25.756982, \n", + "\n", + "\texternal_links = [], \n", + "\n", + "\tkitem_apps = [\n", + "\t\t{\n", + "\t\t\tkitem_app_id: 22,\n", + "\t\t\texecutable: testapp2,\n", + "\t\t\ttitle: data2rdf,\n", + "\t\t\tdescription: None,\n", + "\t\t\ttags: None,\n", + "\t\t\tadditional_properties: {triggerUponUpload: True, triggerUponUploadFileExtensions: ['.csv']}\n", + "\t\t}\n", + "\t], \n", + "\n", + "\tsummary = None, \n", + "\n", + "\tuser_groups = [], \n", + "\n", + "\tcustom_properties = None, \n", + "\n", + "\tdataframe = [\n", + "\t\t{\n", + "\t\t\tcolumn_id: 0,\n", + "\t\t\tname: TestTime\n", + "\t\t}, \n", + "\t\t{\n", + "\t\t\tcolumn_id: 1,\n", + "\t\t\tname: StandardForce\n", + "\t\t}, \n", + "\t\t{\n", + "\t\t\tcolumn_id: 2,\n", + "\t\t\tname: AbsoluteCrossheadTravel\n", + "\t\t}\n", + "\t], \n", + "\n", + "\trdf_exists = True\n", + ")\n" + ] + } + ], + "source": [ + "print(item)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And also that the RDF generation was successful:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@prefix csvw: .\n", + "@prefix dcat: .\n", + "@prefix dcterms: .\n", + "@prefix ns1: .\n", + "@prefix ns2: .\n", + "@prefix rdfs: .\n", + "@prefix xsd: .\n", + "\n", + " a dcat:Dataset ;\n", + " dcterms:hasPart ;\n", + " dcat:distribution [ a dcat:Distribution ;\n", + " dcat:accessURL \"https://bue.materials-data.space/api/knowledge/data_api/58683a2d-7823-4638-bc8e-91b461afa593\"^^xsd:anyURI ;\n", + " dcat:mediaType \"http://www.iana.org/assignments/media-types/text/csv\"^^xsd:anyURI ] .\n", + "\n", + " a ;\n", + " ns1:hasUnit \"http://qudt.org/vocab/unit/MilliM\"^^xsd:anyURI .\n", + "\n", + " a ;\n", + " ns1:hasUnit \"http://qudt.org/vocab/unit/KiloN\"^^xsd:anyURI .\n", + "\n", + " a ;\n", + " ns1:hasUnit \"http://qudt.org/vocab/unit/SEC\"^^xsd:anyURI .\n", + "\n", + " a csvw:TableGroup ;\n", + " csvw:table [ a csvw:Table ;\n", + " rdfs:label \"Time series data\" ;\n", + " csvw:tableSchema [ a csvw:Schema ;\n", + " csvw:column [ a csvw:Column ;\n", + " ns1:quantity ;\n", + " csvw:titles \"C\"^^xsd:string ;\n", + " ns2:page [ a ns2:Document ;\n", + " dcterms:format \"https://www.iana.org/assignments/media-types/application/json\"^^xsd:anyURI ;\n", + " dcterms:identifier \"https://bue.materials-data.space/api/knowledge/data_api/column-2\"^^xsd:anyURI ;\n", + " dcterms:type \"http://purl.org/dc/terms/Dataset\"^^xsd:anyURI ] ],\n", + " [ a csvw:Column ;\n", + " ns1:quantity ;\n", + " csvw:titles \"B\"^^xsd:string ;\n", + " ns2:page [ a ns2:Document ;\n", + " dcterms:format \"https://www.iana.org/assignments/media-types/application/json\"^^xsd:anyURI ;\n", + " dcterms:identifier \"https://bue.materials-data.space/api/knowledge/data_api/column-1\"^^xsd:anyURI ;\n", + " dcterms:type \"http://purl.org/dc/terms/Dataset\"^^xsd:anyURI ] ],\n", + " [ a csvw:Column ;\n", + " ns1:quantity ;\n", + " csvw:titles \"A\"^^xsd:string ;\n", + " ns2:page [ a ns2:Document ;\n", + " dcterms:format \"https://www.iana.org/assignments/media-types/application/json\"^^xsd:anyURI ;\n", + " dcterms:identifier \"https://bue.materials-data.space/api/knowledge/data_api/column-0\"^^xsd:anyURI ;\n", + " dcterms:type \"http://purl.org/dc/terms/Dataset\"^^xsd:anyURI ] ] ] ] .\n", + "\n", + "\n" + ] + } + ], + "source": [ + "print(item.subgraph.serialize())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And now we are able to convert our data into any compatiable unit we want. For the `StandardForce`, it was previously `kN`, but we want to have it in `N` now:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[1300.0,\n", + " 1800.0,\n", + " 2100.0,\n", + " 2600.0,\n", + " 3200.0,\n", + " 3700.0,\n", + " 4300.0,\n", + " 4800.0,\n", + " 5300.0,\n", + " 6000.0]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item.dataframe.StandardForce.convert_to(\"N\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 6.2.2.3 Manipulate dataframe" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to retrieve the dataframe as pd.DataFrame:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TestTimeStandardForceAbsoluteCrossheadTravel
01.21.31.5
11.71.81.9
22.02.12.3
32.52.62.8
43.03.23.4
53.63.73.9
64.14.34.4
74.74.85.0
85.25.35.5
95.86.06.1
\n", + "
" + ], + "text/plain": [ + " TestTime StandardForce AbsoluteCrossheadTravel\n", + "0 1.2 1.3 1.5\n", + "1 1.7 1.8 1.9\n", + "2 2.0 2.1 2.3\n", + "3 2.5 2.6 2.8\n", + "4 3.0 3.2 3.4\n", + "5 3.6 3.7 3.9\n", + "6 4.1 4.3 4.4\n", + "7 4.7 4.8 5.0\n", + "8 5.2 5.3 5.5\n", + "9 5.8 6.0 6.1" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "item.dataframe.to_df()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to overwrite the dataframe with new data:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "item.dataframe = {\n", + " \"TestTime\": list(range(100)),\n", + " \"StandardForce\": list(range(1,101)),\n", + " \"AbsoluteCrossheadTravel\": list(range(2,102))\n", + "}\n", + "dsms.commit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to retrieve the data colum-wise:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "column: TestTime ,\n", + " data: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]\n", + "column: StandardForce ,\n", + " data: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100]\n", + "column: AbsoluteCrossheadTravel ,\n", + " data: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101]\n" + ] + } + ], + "source": [ + "for column in item.dataframe:\n", + " print(\"column:\", column.name, \",\\n\", \"data:\", column.get())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... and also to modify the dataframe directly as we need:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "new_df = item.dataframe.to_df().drop(['TestTime'], axis=1)\n", + "item.dataframe = new_df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 6.2.2.4 Run app on demand" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to run the app on demand, not being triggered automatically during the upload of an attachment every time. For this purpose, we just need to refer to the name of the app we assigned during the KItem creation ( here it is simply `data2rdf`).\n", + "\n", + "Additionally, we need to tell the `attachment_name` and hand over the access token and host url to the app by explicitly setting `set_token` and `set_host_url` to `True`.\n", + "\n", + "The app is running synchronously, hence the `job` is created when the pipeline run finished." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "job = item.kitem_apps.by_title[\"data2rdf\"].run(\n", + " attachment_name=item.attachments[0].name,\n", + " set_token=True,\n", + " set_host_url=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to retrieve the job status:" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "JobStatus(phase='Succeeded', estimated_duration=None, finished_at='08/20/2024, 08:05:35', started_at='08/20/2024, 08:05:15', message=None, progress='1/1')" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job.status" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "... and the job logs:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\"[2024-08-20 08:05:23,481 - dsms_data2rdf.main - INFO]: Fetch KItem: \\n KItem(\\n\\n\\tname = my tensile test experiment, \\n\\n\\tid = 58683a2d-7823-4638-bc8e-91b461afa593, \\n\\n\\tktype_id = dataset, \\n\\n\\tin_backend = True, \\n\\n\\tslug = mytensiletestexperiment-58683a2d, \\n\\n\\tannotations = [], \\n\\n\\tattachments = [\\n\\t\\t{\\n\\t\\t\\tname: dummy_data.csv\\n\\t\\t}\\n\\t], \\n\\n\\tlinked_kitems = [], \\n\\n\\taffiliations = [], \\n\\n\\tauthors = [\\n\\t\\t{\\n\\t\\t\\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\\n\\t\\t}\\n\\t], \\n\\n\\tavatar_exists = True, \\n\\n\\tcontacts = [], \\n\\n\\tcreated_at = 2024-08-20 08:04:25.756982, \\n\\n\\tupdated_at = 2024-08-20 08:04:25.756982, \\n\\n\\texternal_links = [], \\n\\n\\tkitem_apps = [\\n\\t\\t{\\n\\t\\t\\tkitem_app_id: 22,\\n\\t\\t\\texecutable: testapp2,\\n\\t\\t\\ttitle: data2rdf,\\n\\t\\t\\tdescription: None,\\n\\t\\t\\ttags: None,\\n\\t\\t\\tadditional_properties: {triggerUponUpload: True, triggerUponUploadFileExtensions: ['.csv']}\\n\\t\\t}\\n\\t], \\n\\n\\tsummary = None, \\n\\n\\tuser_groups = [], \\n\\n\\tcustom_properties = None, \\n\\n\\tdataframe = [\\n\\t\\t{\\n\\t\\t\\tcolumn_id: 0,\\n\\t\\t\\tname: TestTime\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 1,\\n\\t\\t\\tname: StandardForce\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 2,\\n\\t\\t\\tname: AbsoluteCrossheadTravel\\n\\t\\t}\\n\\t], \\n\\n\\trdf_exists = True\\n)\\n[2024-08-20 08:05:23,494 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser arguments: {'metadata_sep': ',', 'metadata_length': '0', 'time_series_sep': ',', 'time_series_header_length': '1', 'drop_na': 'false', 'fillna': ''}\\n[2024-08-20 08:05:23,494 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser: Parser.csv\\n[2024-08-20 08:05:23,494 - dsms_data2rdf.main - INFO]: Run pipeline with the following config: {'base_iri': 'https://bue.materials-data.space/58683a2d-7823-4638-bc8e-91b461afa593', 'data_download_uri': 'https://bue.materials-data.space/api/knowledge/data_api/58683a2d-7823-4638-bc8e-91b461afa593', 'graph_identifier': 'https://bue.materials-data.space/58683a2d-7823-4638-bc8e-91b461afa593', 'separator': '/', 'encoding': 'utf-8'}\\n[2024-08-20 08:05:28,419 - dsms_data2rdf.main - INFO]: Pipeline did detect any metadata. Will not make annotations for KItem\\n[2024-08-20 08:05:28,810 - dsms_data2rdf.main - INFO]: Checking that dataframe is up to date.\\n[2024-08-20 08:05:28,810 - dsms_data2rdf.main - INFO]: Dataframe upload was successful after 0 retries.\\n[2024-08-20 08:05:28,810 - dsms_data2rdf.main - INFO]: Done!\\n\"\n" + ] + } + ], + "source": [ + "print(job.logs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In case we would like to run the job in the background, we simply add a `wait=False`:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "job = item.kitem_apps.by_title[\"data2rdf\"].run(\n", + " attachment_name=item.attachments[0].name,\n", + " set_token=True,\n", + " set_host_url=True,\n", + " wait=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are able to monitor the job status and logs asynchronously:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Running' estimated_duration=None finished_at=None started_at='08/20/2024, 08:05:39' message=None progress='0/1'\n", + "\n", + " Current logs:\n", + "\"\"\n", + "\n", + " Current status:\n", + "phase='Succeeded' estimated_duration=None finished_at='08/20/2024, 08:05:59' started_at='08/20/2024, 08:05:39' message=None progress='1/1'\n", + "\n", + " Current logs:\n", + "\"[2024-08-20 08:05:45,472 - dsms_data2rdf.main - INFO]: Fetch KItem: \\n KItem(\\n\\n\\tname = my tensile test experiment, \\n\\n\\tid = 58683a2d-7823-4638-bc8e-91b461afa593, \\n\\n\\tktype_id = dataset, \\n\\n\\tin_backend = True, \\n\\n\\tslug = mytensiletestexperiment-58683a2d, \\n\\n\\tannotations = [], \\n\\n\\tattachments = [\\n\\t\\t{\\n\\t\\t\\tname: dummy_data.csv\\n\\t\\t}\\n\\t], \\n\\n\\tlinked_kitems = [], \\n\\n\\taffiliations = [], \\n\\n\\tauthors = [\\n\\t\\t{\\n\\t\\t\\tuser_id: 7f0e5a37-353b-4bbc-b1f1-b6ad575f562d\\n\\t\\t}\\n\\t], \\n\\n\\tavatar_exists = True, \\n\\n\\tcontacts = [], \\n\\n\\tcreated_at = 2024-08-20 08:04:25.756982, \\n\\n\\tupdated_at = 2024-08-20 08:04:25.756982, \\n\\n\\texternal_links = [], \\n\\n\\tkitem_apps = [\\n\\t\\t{\\n\\t\\t\\tkitem_app_id: 22,\\n\\t\\t\\texecutable: testapp2,\\n\\t\\t\\ttitle: data2rdf,\\n\\t\\t\\tdescription: None,\\n\\t\\t\\ttags: None,\\n\\t\\t\\tadditional_properties: {triggerUponUpload: True, triggerUponUploadFileExtensions: ['.csv']}\\n\\t\\t}\\n\\t], \\n\\n\\tsummary = None, \\n\\n\\tuser_groups = [], \\n\\n\\tcustom_properties = None, \\n\\n\\tdataframe = [\\n\\t\\t{\\n\\t\\t\\tcolumn_id: 0,\\n\\t\\t\\tname: TestTime\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 1,\\n\\t\\t\\tname: StandardForce\\n\\t\\t}, \\n\\t\\t{\\n\\t\\t\\tcolumn_id: 2,\\n\\t\\t\\tname: AbsoluteCrossheadTravel\\n\\t\\t}\\n\\t], \\n\\n\\trdf_exists = True\\n)\\n[2024-08-20 08:05:45,485 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser arguments: {'metadata_sep': ',', 'metadata_length': '0', 'time_series_sep': ',', 'time_series_header_length': '1', 'drop_na': 'false', 'fillna': ''}\\n[2024-08-20 08:05:45,485 - dsms_data2rdf.main - INFO]: Run pipeline with the following parser: Parser.csv\\n[2024-08-20 08:05:45,485 - dsms_data2rdf.main - INFO]: Run pipeline with the following config: {'base_iri': 'https://bue.materials-data.space/58683a2d-7823-4638-bc8e-91b461afa593', 'data_download_uri': 'https://bue.materials-data.space/api/knowledge/data_api/58683a2d-7823-4638-bc8e-91b461afa593', 'graph_identifier': 'https://bue.materials-data.space/58683a2d-7823-4638-bc8e-91b461afa593', 'separator': '/', 'encoding': 'utf-8'}\\n[2024-08-20 08:05:50,422 - dsms_data2rdf.main - INFO]: Pipeline did detect any metadata. Will not make annotations for KItem\\n[2024-08-20 08:05:50,816 - dsms_data2rdf.main - INFO]: Checking that dataframe is up to date.\\n[2024-08-20 08:05:50,816 - dsms_data2rdf.main - INFO]: Dataframe upload was successful after 0 retries.\\n[2024-08-20 08:05:50,816 - dsms_data2rdf.main - INFO]: Done!\\n\"\n" + ] + } + ], + "source": [ + "while True:\n", + " time.sleep(1)\n", + " print(\"\\n Current status:\")\n", + " print(job.status)\n", + " print(\"\\n Current logs:\")\n", + " print(job.logs)\n", + " if job.status.phase != \"Running\":\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**IMPORTANT**: When job has run asychronously (in the background), we need to manually refresh the KItem afterwards:" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "item.refresh()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Clean up the DSMS from the tutorial" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "del dsms[item]\n", + "del dsms[appspec]\n", + "dsms.commit()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/testfile.txt b/examples/tutorials/testfile.txt new file mode 100644 index 0000000..7976166 --- /dev/null +++ b/examples/tutorials/testfile.txt @@ -0,0 +1 @@ +This is a calibration protocol! diff --git a/setup.cfg b/setup.cfg index 88fd981..082febd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dsms_sdk -version = v1.3.4 +version = v1.4.0rc25 description = Python SDK core-package for working with the Dataspace Management System (DSMS). long_description = file: README.md long_description_content_type = text/markdown @@ -19,21 +19,44 @@ classifiers = [options] packages = find: install_requires = + PyYAML>=6,<7 + click>=8,<9 + html5lib>=1,<2 + lru-cache<1 pandas>=2,<3 pydantic>=2 pydantic-settings + python-dotenv + qrcode-artistic>=3,<4 rdflib>=6,<7 requests + segno>=1.6,<1.7 python_requires = >=3.8 include_package_data = True +[options.entry_points] +console_scripts = + kitem = dsms.knowledge.cli:lookup_kitem + [options.extras_require] dev = bumpver==2021.1114 dunamai==1.7.0 +docs = + ipython==8.26.0 + jupyter==1.0.0 + myst-parser==4.0.0 + nbsphinx==0.9.5 + sphinx-autobuild==2024.4.16 + sphinx-book-theme==1.1.3 + sphinx-copybutton==0.5.2 + sphinx-markdown-tables==0.0.17 + sphinx-panels==0.4.1 + sphinxcontrib-plantuml==0.30 + sphinxcontrib-redoc==1.6.0 pre_commit = pre-commit==3.3.2 - pylint + pylint==3.2.0 tests = pytest==6.2.5 pytest-mock diff --git a/tests/conftest.py b/tests/conftest.py index ec587f4..e2f16d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import responses if TYPE_CHECKING: - from typing import Any, Dict + from typing import Any, Dict, List class MockDB: @@ -30,9 +30,17 @@ class MockDB: "name": "bar123", "id": "698acdc5-dd97-4217-906e-2c0b44248c17", }, + "cc834fb6-d4cd-43c1-b4b8-4c720ac6f038": { + "name": "bar456", + "id": "cc834fb6-d4cd-43c1-b4b8-4c720ac6f038", + }, } - hdf5 = { + slugs = [ + value.get("name") + "-" + key[:8] for key, value in kitems.items() + ] + + dataframe = { "698acdc5-dd97-4217-906e-2c0b44248c17": [ { "column_id": 0, @@ -66,6 +74,16 @@ def mock_responses(custom_address) -> "Dict[str, Any]": } +@pytest.fixture(scope="function") +def passthru() -> "List[str]": + return [ + "https://qudt.org/2.1/vocab/unit", + "http://qudt.org/2.1/vocab/unit", + "http://qudt.org/vocab/quantitykind", + "https://qudt.org/vocab/quantitykind", + ] + + @pytest.fixture(scope="function") def mock_callbacks(custom_address) -> "Dict[str, Any]": ktypes = urljoin(custom_address, "api/knowledge-type/") @@ -75,11 +93,10 @@ def return_ktypes(request): { "name": "organization", "id": "organization", - "data_schema": { - "title": "Organization", - "type": "object", - "properties": {}, - }, + }, + { + "name": "dataset", + "id": "dataset", }, ] header = {"content_type": "application/json"} @@ -97,17 +114,25 @@ def return_kitems(request): else: return 200, {}, json.dumps(MockDB.kitems[item_id]) - def return_hdf5(request): + def return_dataframe(request): # Extract 'id' parameter from the URL url_parts = request.url.split("/") item_id = url_parts[-1] # Your logic to generate a dynamic response based on 'item_id' # This is just a placeholder; you should replace it with your actual logic - if item_id not in MockDB.hdf5: + if item_id not in MockDB.dataframe: return 404, {}, "KItem does not exist" else: - return 200, {}, json.dumps(MockDB.hdf5[item_id]) + return 200, {}, json.dumps(MockDB.dataframe[item_id]) + + def return_slugs(request): + url_parts = request.url.split("/") + slug = url_parts[-1] + if slug not in MockDB.slugs: + return 404, {}, "Slug does not exist" + else: + return 200, {}, "Slug exists" def _get_kitems() -> "Dict[str, Any]": return { @@ -123,20 +148,36 @@ def _get_kitems() -> "Dict[str, Any]": for uid in MockDB.kitems } - def _get_hdf5() -> "Dict[str, Any]": + def _get_dataframe() -> "Dict[str, Any]": return { urljoin(custom_address, f"api/knowledge/data_api/{uid}"): [ { "method": responses.GET, "returns": { "content_type": "application/json", - "callback": return_hdf5, + "callback": return_dataframe, }, } ] for uid in MockDB.kitems } + def _get_slugs() -> "Dict[str, Any]": + return { + urljoin( + custom_address, f"api/knowledge/kitems/organization/{slug}" + ): [ + { + "method": responses.HEAD, + "returns": { + "content_type": "application/json", + "callback": return_slugs, + }, + } + ] + for slug in MockDB.slugs + } + return { ktypes: [ { @@ -148,12 +189,17 @@ def _get_hdf5() -> "Dict[str, Any]": } ], **_get_kitems(), - **_get_hdf5(), + **_get_dataframe(), + **_get_slugs(), } @pytest.fixture(autouse=True, scope="function") -def register_mocks(mock_responses, mock_callbacks, custom_address) -> str: +def register_mocks( + mock_responses, mock_callbacks, passthru, custom_address +) -> str: + for url in passthru: + responses.add_passthru(url) for url, endpoints in mock_responses.items(): for response in endpoints: responses.add( diff --git a/tests/test_kitem.py b/tests/test_kitem.py index eea192a..b1a1cdf 100644 --- a/tests/test_kitem.py +++ b/tests/test_kitem.py @@ -23,6 +23,8 @@ def test_kitem_basic(custom_address, get_mock_kitem_ids): ktype_id=dsms.ktypes.Organization, ) + assert instance.is_a(dsms.ktypes.Organization) + assert isinstance(instance.dsms, DSMS) assert isinstance(instance.dsms.config, Configuration) assert Context.dsms == instance.dsms @@ -49,6 +51,8 @@ def test_kitem_config_class(custom_address, get_mock_kitem_ids): ktype_id=dsms.ktypes.Organization, ) + assert instance.is_a(dsms.ktypes.Organization) + assert isinstance(instance.dsms, DSMS) assert config == instance.dsms.config assert Context.dsms == instance.dsms @@ -82,9 +86,11 @@ def test_kitem_custom_config_env(custom_address, get_mock_kitem_ids): custom_instance = KItem( id=get_mock_kitem_ids[0], name="foo123", - ktype_id=dsms.ktypes.Organization, + ktype_id=dsms.ktypes.Dataset, ) + assert custom_instance.is_a(dsms.ktypes.Dataset) + assert str(custom_instance.dsms.config.host_url) == custom_address os.environ.pop("DSMS_HOST_URL") @@ -140,7 +146,7 @@ def test_kitem_default_ktypes(custom_address): with pytest.warns(UserWarning, match="No authentication details"): dsms = DSMS(host_url=custom_address) - assert len(dsms.ktypes) == 1 + assert len(dsms.ktypes) == 2 @responses.activate @@ -159,3 +165,34 @@ def test_ktype_property(get_mock_kitem_ids, custom_address): ) assert kitem.ktype == Context.ktypes.get(dsms.ktypes.Organization.value) + + +# @responses.activate +# def test_ktype_custom_property_assignment(get_mock_kitem_ids, custom_address): +# from dsms.knowledge.properties.custom_datatype.numerical import NumericalDataType +# from dsms.core.dsms import DSMS +# from dsms.knowledge.kitem import KItem + +# with pytest.warns(UserWarning, match="No authentication details"): +# dsms = DSMS(host_url=custom_address) + +# kitem = KItem( +# id=get_mock_kitem_ids[0], +# name="foo123", +# ktype_id=dsms.ktypes.Organization, +# custom_properties={"material": "abcd", "tester": 123} +# ) + +# assert kitem.custom_properties.material == "abcd" +# assert kitem.custom_properties.tester == 123 +# assert isinstance(kitem.custom_properties.tester, NumericalDataType) + +# kitem.custom_properties = {"material": "def", "tester2": 123} + +# assert kitem.custom_properties.material == "def" +# assert kitem.custom_properties.tester2 == 123 +# assert isinstance(kitem.custom_properties.tester2, NumericalDataType) + +# kitem.custom_properties.tester2 = 456 +# assert kitem.custom_properties.tester2 == 456 +# assert isinstance(kitem.custom_properties.tester2, NumericalDataType) diff --git a/tests/test_utils.py b/tests/test_utils.py index da3d36a..485c474 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -51,35 +51,58 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address): name="bar123", ) - annotation = { - "iri": "http://example.org/", - "name": "foo", - "namespace": "example", - } - annotation2 = { - "iri": "http://example.org/", - "name": "bar", - "namespace": "example", - } user_group = {"name": "private", "group_id": "private_123"} - app = {"executable": "foo.exe"} - app2 = {"executable": "bar.exe"} - - kitem_old = KItem( - id=get_mock_kitem_ids[0], - name="foo123", - ktype_id=dsms.ktypes.Organization, - annotations=[annotation2], - linked_kitems=[linked_kitem1, linked_kitem2], - user_groups=[user_group], - kitem_apps=[app2], - ) + app = {"executable": "foo.exe", "title": "foo"} + + kitem_old = { + "id": get_mock_kitem_ids[0], + "name": "foo123", + "ktype_id": dsms.ktypes.Organization.value, + "annotations": [ + { + "iri": "http://example.org/", + "name": "bar", + "namespace": "example", + "description": None, + } + ], + "linked_kitems": [ + { + "id": str(get_mock_kitem_ids[1]), + "ktype_id": dsms.ktypes.Organization.value, + "name": "foo456", + }, + { + "id": str(get_mock_kitem_ids[2]), + "ktype_id": dsms.ktypes.Organization.value, + "name": "foo789", + }, + ], + "user_groups": [user_group], + "kitem_apps": [ + { + "id": get_mock_kitem_ids[0], + "kitem_app_id": 17, + "executable": "bar.exe", + "title": "bar", + "description": None, + "tags": None, + "additional_properties": None, + } + ], + } kitem_new = KItem( id=get_mock_kitem_ids[0], name="foo123", ktype_id=dsms.ktypes.Organization, - annotations=[annotation], + annotations=[ + { + "iri": "http://example.org/", + "name": "foo", + "namespace": "example", + } + ], linked_kitems=[linked_kitem3], user_groups=[user_group], kitem_apps=[app], @@ -87,25 +110,78 @@ def test_kitem_diffs(get_mock_kitem_ids, custom_address): expected = { "kitems_to_link": [ - obj.model_dump() for obj in kitem_new.linked_kitems + {"id": str(obj.id)} for obj in kitem_new.linked_kitems ], "annotations_to_link": [ - obj.model_dump() for obj in kitem_new.annotations + { + "iri": "http://example.org/", + "name": "foo", + "namespace": "example", + "description": None, + } ], "user_groups_to_add": [], "kitem_apps_to_update": [ - obj.model_dump() for obj in kitem_new.kitem_apps + { + "executable": "foo.exe", + "title": "foo", + "description": None, + "tags": None, + "additional_properties": None, + } ], "kitems_to_unlink": [ - obj.model_dump() for obj in kitem_old.linked_kitems + {"id": str(linked.id)} for linked in [linked_kitem1, linked_kitem2] ], "annotations_to_unlink": [ - obj.model_dump() for obj in kitem_old.annotations + { + "iri": "http://example.org/", + "name": "bar", + "namespace": "example", + "description": None, + } ], "user_groups_to_remove": [], "kitem_apps_to_remove": [ - obj.model_dump() for obj in kitem_old.kitem_apps + { + "executable": "bar.exe", + "title": "bar", + "description": None, + "tags": None, + "additional_properties": None, + } ], } diffs = _get_kitems_diffs(kitem_old, kitem_new) - assert sorted(diffs) == sorted(expected) + + for key, value in diffs.items(): + assert value == expected.pop(key) + assert len(expected) == 0 + + +@responses.activate +def test_unit_conversion(custom_address): + """Test unit conversion test""" + from dsms import DSMS + from dsms.knowledge.semantics.units import get_conversion_factor + + with pytest.warns(UserWarning, match="No authentication details"): + DSMS(host_url=custom_address) + + assert get_conversion_factor("mm", "m", decimals=3) == 0.001 + + assert get_conversion_factor("km", "in", decimals=1) == 39370.1 + + assert get_conversion_factor("GPa", "MPa") == 1000 + + assert ( + get_conversion_factor( + "http://qudt.org/vocab/unit/M", + "http://qudt.org/vocab/unit/IN", + decimals=1, + ) + == 39.4 + ) + + with pytest.raises(ValueError, match="Unit "): + get_conversion_factor("kPa", "cm")