From d6fa9a1feb7dace89c0c1ac1e8fa21274ca585d4 Mon Sep 17 00:00:00 2001 From: Sierra Taylor Moxon Date: Mon, 15 Jul 2024 10:58:32 -0700 Subject: [PATCH 1/6] port over changes in linkml-runtime/linkml-model/linkml_files.py --- linkml_model/linkml_files.py | 83 ++++++++++++++---- tests/test_linkml_files.py | 158 +++++++++++++++++++++++++++++++++-- 2 files changed, 216 insertions(+), 25 deletions(-) diff --git a/linkml_model/linkml_files.py b/linkml_model/linkml_files.py index 48d11cc6..82060a8b 100644 --- a/linkml_model/linkml_files.py +++ b/linkml_model/linkml_files.py @@ -1,6 +1,8 @@ import os +from dataclasses import dataclass from enum import Enum, auto -from typing import Optional, Union +from pathlib import Path +from typing import Optional, Union, Dict import requests from rdflib import Namespace @@ -34,6 +36,7 @@ class Source(_AutoName): class Format(Enum): """ LinkML package formats """ + EXCEL = "xlsx" GRAPHQL = "graphql" HTML = "" JSON = "json" @@ -44,30 +47,76 @@ class Format(Enum): NATIVE_SHEXC = "model.shex" NATIVE_SHEXJ = "model.shexj" OWL = "owl.ttl" + PREFIXMAP = "yaml" + PROTOBUF = "proto" PYTHON = "py" RDF = "ttl" + SHACL = "shacl.ttl" SHEXC = "shex" SHEXJ = "shexj" + SQLDDL = "sql" + SQLSCHEMA = "sql" YAML = "yaml" -class _Path(Enum): +META_ONLY = ( + Format.EXCEL, + Format.GRAPHQL, + Format.OWL, + Format.PREFIXMAP, + Format.PROTOBUF, + Format.SHACL, + Format.SQLDDL, + Format.SQLSCHEMA +) + + +@dataclass +class FormatPath: + path: str + extension: str + + def model_path(self, model: str) -> Path: + return (Path(self.path) / model).with_suffix(self.extension) + + +class _Path: """ LinkML Relative paths""" - GRAPHQL = "graphql" - HTML = "docs" - JSON = "json" - JSONLD = "jsonld" - JSON_SCHEMA = "jsonschema" - NATIVE_JSONLD = "jsonld" - NATIVE_RDF = "ttl" - NATIVE_SHEXC = "shex" - NATIVE_SHEXJ = "shex" - OWL = "owl" - PYTHON = "linkml_model" - RDF = "rdf" - SHEXC = "shex" - SHEXJ = "shex" - YAML = "model/schema" + EXCEL = FormatPath("excel", "xlsx") + GRAPHQL = FormatPath("graphql", "graphql") + HTML = FormatPath("", "html") + JSON = FormatPath("json", "json") + JSONLD = FormatPath("jsonld", "context.jsonld") + JSON_SCHEMA = FormatPath("jsonschema", "schema.json") + NATIVE_JSONLD = FormatPath("jsonld", "context.jsonld") + NATIVE_RDF = FormatPath("rdf", "ttl") + NATIVE_SHEXC = FormatPath("shex", "shex") + NATIVE_SHEXJ = FormatPath("shex", "shexj") + OWL = FormatPath("owl", "owl.ttl") + PREFIXMAP = FormatPath('prefixmap', 'yaml') + PROTOBUF = FormatPath("protobuf", "proto") + PYTHON = FormatPath("", "py") + RDF = FormatPath("rdf", "ttl") + SHACL = FormatPath("shacl", "shacl.ttl") + SHEXC = FormatPath("shex", "shex") + SHEXJ = FormatPath("shex", "shexj") + SQLDDL = FormatPath("sqlddl", "sql") + SQLSCHEMA = FormatPath("sqlschema", "sql") + YAML = FormatPath(str(Path("model") / "schema"), "yaml") + + @classmethod + def items(cls) -> Dict[str, FormatPath]: + return {k: v for k, v in cls.__dict__.items() if not k.startswith('_')} + + @classmethod + def get(cls, item: Union[str, Format]) -> FormatPath: + if isinstance(item, Format): + item = item.name.upper() + return getattr(cls, item) + + def __class_getitem__(cls, item: str) -> FormatPath: + return getattr(cls, item) + class ReleaseTag(_AutoName): diff --git a/tests/test_linkml_files.py b/tests/test_linkml_files.py index 05b6cae3..65db5384 100644 --- a/tests/test_linkml_files.py +++ b/tests/test_linkml_files.py @@ -1,15 +1,157 @@ -import os -import unittest -import re +import pytest +import requests +from pathlib import Path +from itertools import product +from urllib.parse import urlparse -import linkml_model.linkml_files as fileloc -from linkml_model.linkml_files import URL_FOR, Format, Source, LOCAL_PATH_FOR, GITHUB_IO_PATH_FOR, GITHUB_PATH_FOR, \ +try: + import requests_cache + HAVE_REQUESTS_CACHE = True +except ImportError: + HAVE_REQUESTS_CACHE = False + +from linkml_runtime.linkml_model.linkml_files import ( + Source, + Format, + _Path, + URL_FOR, + LOCAL_PATH_FOR, + LOCAL_BASE, + GITHUB_IO_PATH_FOR, + GITHUB_PATH_FOR, + META_ONLY, ReleaseTag -from tests import abspath +) + +EXPECTED_FORMATS = [ + (source, fmt) for source, fmt in product(Source, Format) + if (fmt not in META_ONLY or source == Source.META) +] + +W3ID_EXTENSIONS = ( + 'xlsx', + 'html', + 'yaml', + 'graphql', + 'context.json', + 'context.jsonld', + 'schema.json', + 'json', + 'ttl', + 'owl', + 'prefixmap', + 'shexj', + 'proto', + 'shacl.ttl', + 'shex', + 'shexc', + 'shexj', + 'sql', +) +W3ID_FORMATS = [ + (source, fmt) for source, fmt in EXPECTED_FORMATS + if _Path.get(fmt.name).extension in W3ID_EXTENSIONS +] +"""The formats that have rewrite rules at https://github.com/perma-id/w3id.org/blob/master/linkml/.htaccess""" + +@pytest.mark.parametrize( + 'source,fmt', + EXPECTED_FORMATS +) +def test_local_paths(source, fmt): + a_path = Path(LOCAL_PATH_FOR(source, fmt)) + assert a_path.exists() + assert a_path.is_absolute() + +@pytest.mark.parametrize( + 'fmt', + Format.__iter__() +) +def test_format_paths(fmt): + """Every format should have an entry in _Path""" + assert fmt.name in _Path.items() + +def test_no_unmapped_dirs(): + """ + There should be no additional directories that don't have a mapping for Format. + """ + EXCLUDES = ('__pycache__',) + + expected = {LOCAL_BASE / _Path.get(fmt.name).path for fmt in Format} + expected.add(LOCAL_BASE / 'model') + + actual = {a_dir for a_dir in LOCAL_BASE.iterdir() if a_dir.is_dir() and a_dir.name not in EXCLUDES} + # Special case the root directory + actual.add(LOCAL_BASE) + # Special case YAML which is in a subdirectory - we've checked for existence above + actual.add(LOCAL_BASE / _Path.get('YAML').path) + assert expected == actual + + +# -------------------------------------------------- +# URLs +# -------------------------------------------------- + +@pytest.mark.skip("github paths largely unused and expensive to test due to ratelimiting") +@pytest.mark.parametrize( + 'release_type', + ReleaseTag.__iter__() +) +@pytest.mark.parametrize( + 'source,fmt', + EXPECTED_FORMATS +) +def test_github_path_exists(source,fmt, release_type): + url = GITHUB_PATH_FOR(source, fmt, release_type) + res = requests.get(url) + assert res.status_code != 404, url + + +@pytest.mark.parametrize( + 'release_type', + ReleaseTag.__iter__() +) +@pytest.mark.parametrize( + 'source,fmt', + EXPECTED_FORMATS +) +def test_github_path_format(source,fmt, release_type): + if release_type == ReleaseTag.CURRENT: + pytest.skip("Need to cache network requests for this") + + url = GITHUB_PATH_FOR(source, fmt, release_type) + # ensure it parses + assert urlparse(url) + # for windows... + assert '\\' not in url + +@pytest.mark.skip("github paths largely unused") +@pytest.mark.parametrize( + 'source,fmt', + EXPECTED_FORMATS +) +def test_github_io_path(source,fmt): + url = GITHUB_IO_PATH_FOR(source, fmt) + res = requests.get(url) + assert res.status_code != 404, url + -root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "linkml_model")) +@pytest.mark.skipif(not HAVE_REQUESTS_CACHE,reason= 'Need to cache this') +@pytest.mark.parametrize( + 'source,fmt', + W3ID_FORMATS +) +def test_url_for_format(source,fmt): + url = URL_FOR(source, fmt) + res = requests.get(url) + assert res.status_code != 404, url -SKIP_GITHUB_API = False # True means don't do the github API tests +def test_fixed_meta_url(): + """ + One fixed canary value - the METAMODEL_URI as used in linkml main shouldn't change + """ + assert URL_FOR(Source.META, Format.YAML) == 'https://w3id.org/linkml/meta.yaml' + assert URL_FOR(Source.META, Format.JSONLD) == 'https://w3id.org/linkml/meta.context.jsonld' class LinkMLFilesTestCase(unittest.TestCase): From db2f648d240e81288e104fda7e2ade4cf9f17eab Mon Sep 17 00:00:00 2001 From: Sierra Taylor Moxon Date: Mon, 15 Jul 2024 16:54:39 -0700 Subject: [PATCH 2/6] port linkml-runtime test changes to linkml-model so that make model-update in linkml-runtime stays the same --- .github/workflows/main.yaml | 3 +- Makefile | 2 +- linkml_model/linkml_files.py | 6 +- poetry.lock | 132 +++++------------------------------ pyproject.toml | 2 + tests/test_linkml_files.py | 77 ++++---------------- 6 files changed, 38 insertions(+), 184 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8c41667d..a0a6207f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -15,7 +15,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: diff --git a/Makefile b/Makefile index 458eb09b..395b6f88 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ test-schema: $(RUN) gen-project -d tmp $(SOURCE_SCHEMA_PATH) test-python: - $(RUN) python -m unittest discover + $(RUN) python -m pytest # TODO: switch to linkml-run-examples when normalize is implemented test-examples: $(SOURCE_SCHEMA_PATH) diff --git a/linkml_model/linkml_files.py b/linkml_model/linkml_files.py index 82060a8b..a8fb89b1 100644 --- a/linkml_model/linkml_files.py +++ b/linkml_model/linkml_files.py @@ -17,8 +17,6 @@ GITHUB_TAGS = GITHUB_BASE + "tags" - - class _AutoName(Enum): @staticmethod def _generate_next_value_(name, start, count, last_values): @@ -118,7 +116,6 @@ def __class_getitem__(cls, item: str) -> FormatPath: return getattr(cls, item) - class ReleaseTag(_AutoName): """ Release tags LATEST - the absolute latest in the supplied branch @@ -203,7 +200,8 @@ def url(self) -> str: def file(self) -> str: return LOCAL_PATH_FOR(self._model, self._format) - def github_loc(self, tag: Optional[str] = None, branch: Optional[str] = None, release: ReleaseTag = None) -> str: + def github_loc(self, tag: Optional[str] = None, branch: Optional[str] = None, + release: ReleaseTag = None) -> str: if not tag and not branch and not release: return GITHUB_IO_PATH_FOR(self._model, self._format) if tag: diff --git a/poetry.lock b/poetry.lock index fcd6ed7c..e6b99ee0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "antlr4-python3-runtime" version = "4.9.3" description = "ANTLR 4.9.3 runtime for Python 3.7" -category = "dev" optional = false python-versions = "*" files = [ @@ -15,7 +14,6 @@ files = [ name = "arrow" version = "1.2.3" description = "Better dates & times for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -30,7 +28,6 @@ python-dateutil = ">=2.7.0" name = "attrs" version = "22.2.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -49,7 +46,6 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy name = "beautifulsoup4" version = "4.12.0" description = "Screen-scraping library" -category = "dev" optional = false python-versions = ">=3.6.0" files = [ @@ -68,7 +64,6 @@ lxml = ["lxml"] name = "certifi" version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -80,7 +75,6 @@ files = [ name = "cfgraph" version = "0.2.1" description = "rdflib collections flattening graph" -category = "dev" optional = false python-versions = "*" files = [ @@ -94,7 +88,6 @@ rdflib = ">=0.4.2" name = "chardet" version = "5.1.0" description = "Universal encoding detector for Python 3" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -106,7 +99,6 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -191,7 +183,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -206,7 +197,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "codespell" version = "2.2.5" description = "Codespell" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -224,7 +214,6 @@ types = ["chardet (>=5.1.0)", "mypy", "pytest", "pytest-cov", "pytest-dependency name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -236,7 +225,6 @@ files = [ name = "curies" version = "0.5.5" description = "Idiomatic conversion between URIs and compact URIs (CURIEs)." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -262,7 +250,6 @@ tests = ["coverage", "pytest"] name = "decorator" version = "5.1.1" description = "Decorators for Humans" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -274,7 +261,6 @@ files = [ name = "deprecated" version = "1.2.13" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -292,7 +278,6 @@ dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version name = "editorconfig" version = "0.12.3" description = "EditorConfig File Locator and Interpreter for Python" -category = "dev" optional = false python-versions = "*" files = [ @@ -304,7 +289,6 @@ files = [ name = "et-xmlfile" version = "1.1.0" description = "An implementation of lxml.xmlfile for the standard library" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -316,7 +300,6 @@ files = [ name = "exceptiongroup" version = "1.1.1" description = "Backport of PEP 654 (exception groups)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -331,7 +314,6 @@ test = ["pytest (>=6)"] name = "fqdn" version = "1.5.1" description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers" -category = "dev" optional = false python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4" files = [ @@ -343,7 +325,6 @@ files = [ name = "ghp-import" version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." -category = "dev" optional = false python-versions = "*" files = [ @@ -361,7 +342,6 @@ dev = ["flake8", "markdown", "twine", "wheel"] name = "graphviz" version = "0.20.1" description = "Simple Python interface for Graphviz" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -378,7 +358,6 @@ test = ["coverage", "mock (>=4)", "pytest (>=7)", "pytest-cov", "pytest-mock (>= name = "greenlet" version = "2.0.2" description = "Lightweight in-process concurrent programming" -category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" files = [ @@ -456,7 +435,6 @@ test = ["objgraph", "psutil"] name = "hbreader" version = "0.9.1" description = "Honey Badger reader - a generic file/url/string open and read tool" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -468,7 +446,6 @@ files = [ name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -480,7 +457,6 @@ files = [ name = "importlib-metadata" version = "4.13.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -500,7 +476,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "6.0.1" description = "Read resources from Python packages" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -519,7 +494,6 @@ testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -531,7 +505,6 @@ files = [ name = "isodate" version = "0.6.1" description = "An ISO 8601 date/time/duration parser and formatter" -category = "main" optional = false python-versions = "*" files = [ @@ -546,7 +519,6 @@ six = "*" name = "isoduration" version = "20.11.0" description = "Operations with ISO 8601 durations" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -561,7 +533,6 @@ arrow = ">=0.15.0" name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -579,7 +550,6 @@ i18n = ["Babel (>=2.7)"] name = "jsbeautifier" version = "1.14.7" description = "JavaScript unobfuscator and beautifier." -category = "dev" optional = false python-versions = "*" files = [ @@ -594,7 +564,6 @@ six = ">=1.13.0" name = "json-flattener" version = "0.1.9" description = "Python library for denormalizing nested dicts or json objects to tables and back" -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -610,7 +579,6 @@ pyyaml = "*" name = "jsonasobj" version = "1.3.1" description = "JSON as python objects" -category = "dev" optional = false python-versions = "*" files = [ @@ -622,7 +590,6 @@ files = [ name = "jsonasobj2" version = "1.0.4" description = "JSON as python objects - version 2" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -637,7 +604,6 @@ hbreader = "*" name = "jsonpatch" version = "1.32" description = "Apply JSON-Patches (RFC 6902)" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -652,7 +618,6 @@ jsonpointer = ">=1.9" name = "jsonpath-ng" version = "1.5.3" description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." -category = "dev" optional = false python-versions = "*" files = [ @@ -670,7 +635,6 @@ six = "*" name = "jsonpointer" version = "2.3" description = "Identify specific nodes in a JSON document (RFC 6901)" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -682,7 +646,6 @@ files = [ name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -710,7 +673,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "linkml" version = "1.6.11" description = "Linked Open Data Modeling Language" -category = "dev" optional = false python-versions = ">=3.8.1,<4.0.0" files = [ @@ -748,7 +710,6 @@ watchdog = ">=0.9.0" name = "linkml-dataops" version = "0.1.0" description = "LinkML Data Operations API" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -767,7 +728,6 @@ linkml-runtime = ">=1.1.6" name = "linkml-runtime" version = "1.6.3" description = "Runtime environment for LinkML, the Linked open data modeling language" -category = "main" optional = false python-versions = ">=3.7.6,<4.0.0" files = [ @@ -794,7 +754,6 @@ requests = "*" name = "markdown" version = "3.3.7" description = "Python implementation of Markdown." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -812,7 +771,6 @@ testing = ["coverage", "pyyaml"] name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -872,7 +830,6 @@ files = [ name = "mergedeep" version = "1.3.4" description = "A deep merge function for 🐍." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -884,7 +841,6 @@ files = [ name = "mike" version = "1.2.0.dev0" description = "Manage multiple versions of your MkDocs-powered documentation" -category = "dev" optional = false python-versions = "*" files = [] @@ -912,7 +868,6 @@ resolved_reference = "be1aafe244bc86172fbce52a903c9ab83e2e4a26" name = "mkdocs" version = "1.4.2" description = "Project documentation with Markdown." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -941,7 +896,6 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp name = "mkdocs-material" version = "8.5.11" description = "Documentation that simply works" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -962,7 +916,6 @@ requests = ">=2.26" name = "mkdocs-material-extensions" version = "1.1.1" description = "Extension pack for Python Markdown and MkDocs Material." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -974,7 +927,6 @@ files = [ name = "mkdocs-mermaid2-plugin" version = "0.6.0" description = "A MkDocs plugin for including mermaid graphs in markdown sources" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -996,7 +948,6 @@ setuptools = ">=18.5" name = "openpyxl" version = "3.1.2" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1011,7 +962,6 @@ et-xmlfile = "*" name = "packaging" version = "23.0" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1023,7 +973,6 @@ files = [ name = "parse" version = "1.19.0" description = "parse() is the opposite of format()" -category = "dev" optional = false python-versions = "*" files = [ @@ -1035,7 +984,6 @@ files = [ name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1045,14 +993,13 @@ files = [ [[package]] name = "pluggy" -version = "1.0.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] @@ -1063,7 +1010,6 @@ testing = ["pytest", "pytest-benchmark"] name = "ply" version = "3.11" description = "Python Lex & Yacc" -category = "dev" optional = false python-versions = "*" files = [ @@ -1075,7 +1021,6 @@ files = [ name = "prefixcommons" version = "0.1.12" description = "A python API for working with ID prefixes" -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1093,7 +1038,6 @@ requests = ">=2.28.1,<3.0.0" name = "prefixmaps" version = "0.1.4" description = "A python library for retrieving semantic prefix maps" -category = "main" optional = false python-versions = ">=3.7.6,<4.0.0" files = [ @@ -1109,7 +1053,6 @@ pyyaml = ">=5.3.1" name = "pydantic" version = "1.10.7" description = "Data validation and settings management using python type hints" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1162,7 +1105,6 @@ email = ["email-validator (>=1.0.3)"] name = "pygments" version = "2.14.0" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1177,7 +1119,6 @@ plugins = ["importlib-metadata"] name = "pyjsg" version = "0.11.10" description = "Python JSON Schema Grammar interpreter" -category = "dev" optional = false python-versions = "*" files = [ @@ -1193,7 +1134,6 @@ jsonasobj = ">=1.2.1" name = "pymdown-extensions" version = "9.10" description = "Extension pack for Python Markdown." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1209,7 +1149,6 @@ pyyaml = "*" name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -1224,7 +1163,6 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1261,7 +1199,6 @@ files = [ name = "pyshex" version = "0.8.1" description = "Python ShEx Implementation" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1284,7 +1221,6 @@ urllib3 = "*" name = "pyshexc" version = "0.9.1" description = "PyShExC - Python ShEx compiler" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1302,33 +1238,30 @@ shexjsg = ">=0.8.1" [[package]] name = "pytest" -version = "7.2.2" +version = "8.2.2" description = "pytest: simple powerful testing with Python" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"}, - {file = "pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"}, + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, ] [package.dependencies] -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-logging" version = "2015.11.4" description = "Configures logging and allows tweaking the log level with a py.test flag" -category = "main" optional = false python-versions = "*" files = [ @@ -1342,7 +1275,6 @@ pytest = ">=2.8.1" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1357,10 +1289,10 @@ six = ">=1.5" name = "pytrie" version = "0.4.0" description = "A pure Python implementation of the trie data structure." -category = "main" optional = false python-versions = "*" files = [ + {file = "PyTrie-0.4.0-py3-none-any.whl", hash = "sha256:f687c224ee8c66cda8e8628a903011b692635ffbb08d4b39c5f92b18eb78c950"}, {file = "PyTrie-0.4.0.tar.gz", hash = "sha256:8f4488f402d3465993fb6b6efa09866849ed8cda7903b50647b7d0342b805379"}, ] @@ -1371,7 +1303,6 @@ sortedcontainers = "*" name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1421,7 +1352,6 @@ files = [ name = "pyyaml-env-tag" version = "0.1" description = "A custom YAML tag for referencing environment variables in YAML files. " -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1434,14 +1364,13 @@ pyyaml = "*" [[package]] name = "rdflib" -version = "6.3.2" +version = "7.0.0" description = "RDFLib is a Python library for working with RDF, a simple yet powerful language for representing information." -category = "main" optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.8.1,<4.0.0" files = [ - {file = "rdflib-6.3.2-py3-none-any.whl", hash = "sha256:36b4e74a32aa1e4fa7b8719876fb192f19ecd45ff932ea5ebbd2e417a0247e63"}, - {file = "rdflib-6.3.2.tar.gz", hash = "sha256:72af591ff704f4caacea7ecc0c5a9056b8553e0489dd4f35a9bc52dbd41522e0"}, + {file = "rdflib-7.0.0-py3-none-any.whl", hash = "sha256:0438920912a642c866a513de6fe8a0001bd86ef975057d6962c79ce4771687cd"}, + {file = "rdflib-7.0.0.tar.gz", hash = "sha256:9995eb8569428059b8c1affd26b25eac510d64f5043d9ce8c84e0d0036e995ae"}, ] [package.dependencies] @@ -1458,7 +1387,6 @@ networkx = ["networkx (>=2.0.0,<3.0.0)"] name = "rdflib-jsonld" version = "0.6.1" description = "rdflib extension adding JSON-LD parser and serializer" -category = "dev" optional = false python-versions = "*" files = [ @@ -1473,7 +1401,6 @@ rdflib = ">=5.0.0" name = "rdflib-shim" version = "1.0.3" description = "Shim for rdflib 5 and 6 incompatibilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1489,7 +1416,6 @@ rdflib-jsonld = "0.6.1" name = "requests" version = "2.28.2" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7, <4" files = [ @@ -1511,7 +1437,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rfc3339-validator" version = "0.1.4" description = "A pure python RFC3339 validator" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1526,7 +1451,6 @@ six = "*" name = "rfc3987" version = "1.3.8" description = "Parsing and validation of URIs (RFC 3986) and IRIs (RFC 3987)" -category = "dev" optional = false python-versions = "*" files = [ @@ -1538,7 +1462,6 @@ files = [ name = "ruamel-yaml" version = "0.17.21" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -1557,7 +1480,6 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] name = "ruamel-yaml-clib" version = "0.2.7" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1604,7 +1526,6 @@ files = [ name = "setuptools" version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1621,7 +1542,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "shexjsg" version = "0.8.2" description = "ShExJSG - Astract Syntax Tree for the ShEx 2.0 language" -category = "dev" optional = false python-versions = "*" files = [ @@ -1636,7 +1556,6 @@ pyjsg = ">=0.11.10" name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1648,7 +1567,6 @@ files = [ name = "sortedcontainers" version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" -category = "main" optional = false python-versions = "*" files = [ @@ -1660,7 +1578,6 @@ files = [ name = "soupsieve" version = "2.4" description = "A modern CSS selector implementation for Beautiful Soup." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1672,7 +1589,6 @@ files = [ name = "sparqlslurper" version = "0.5.1" description = "SPARQL Slurper for rdflib" -category = "dev" optional = false python-versions = ">=3.7.4" files = [ @@ -1689,7 +1605,6 @@ sparqlwrapper = ">=1.8.2" name = "sparqlwrapper" version = "2.0.0" description = "SPARQL Endpoint interface to Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1710,7 +1625,6 @@ pandas = ["pandas (>=1.3.5)"] name = "sqlalchemy" version = "2.0.7" description = "Database Abstraction Library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1788,7 +1702,6 @@ sqlcipher = ["sqlcipher3-binary"] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1800,7 +1713,6 @@ files = [ name = "typing-extensions" version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1812,7 +1724,6 @@ files = [ name = "uri-template" version = "1.2.0" description = "RFC 6570 URI Template Processor" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1827,7 +1738,6 @@ dev = ["flake8 (<4.0.0)", "flake8-annotations", "flake8-bugbear", "flake8-commas name = "urllib3" version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -1844,7 +1754,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "verspec" version = "0.1.0" description = "Flexible version handling" -category = "dev" optional = false python-versions = "*" files = [ @@ -1859,7 +1768,6 @@ test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] name = "watchdog" version = "3.0.0" description = "Filesystem events monitoring" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1899,7 +1807,6 @@ watchmedo = ["PyYAML (>=3.10)"] name = "webcolors" version = "1.13" description = "A library for working with the color formats defined by HTML and CSS." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1915,7 +1822,6 @@ tests = ["pytest", "pytest-cov"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2000,7 +1906,6 @@ files = [ name = "yamllint" version = "1.32.0" description = "A linter for YAML files." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2019,7 +1924,6 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2034,4 +1938,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "667465bd4c847c247e8d5eb8fe673f107bfdfe5877d5fe395dd5c6cf769e3ffd" +content-hash = "0dad7cd83ffe02a0442c5a4a752c49e63ce651ec13ab29f70655bb4b9bc33ea1" diff --git a/pyproject.toml b/pyproject.toml index 096cb73f..801ad5e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,11 +33,13 @@ packages = [ [tool.poetry.dependencies] python = "^3.9" linkml-runtime = "^1.6.3" +rdflib = "^7.0.0" [tool.poetry.group.dev.dependencies] linkml = "^1.6.11" codespell = "^2.2.5" yamllint = "^1.32.0" +pytest = "^8.2.2" [tool.poetry.group.docs.dependencies] mkdocs = "^1.4.2" diff --git a/tests/test_linkml_files.py b/tests/test_linkml_files.py index 65db5384..f05bae3b 100644 --- a/tests/test_linkml_files.py +++ b/tests/test_linkml_files.py @@ -6,11 +6,12 @@ try: import requests_cache + HAVE_REQUESTS_CACHE = True except ImportError: HAVE_REQUESTS_CACHE = False -from linkml_runtime.linkml_model.linkml_files import ( +from linkml_model.linkml_files import ( Source, Format, _Path, @@ -29,7 +30,6 @@ ] W3ID_EXTENSIONS = ( - 'xlsx', 'html', 'yaml', 'graphql', @@ -39,14 +39,9 @@ 'json', 'ttl', 'owl', - 'prefixmap', - 'shexj', - 'proto', - 'shacl.ttl', 'shex', 'shexc', - 'shexj', - 'sql', + 'shexj' ) W3ID_FORMATS = [ (source, fmt) for source, fmt in EXPECTED_FORMATS @@ -54,6 +49,7 @@ ] """The formats that have rewrite rules at https://github.com/perma-id/w3id.org/blob/master/linkml/.htaccess""" + @pytest.mark.parametrize( 'source,fmt', EXPECTED_FORMATS @@ -63,6 +59,7 @@ def test_local_paths(source, fmt): assert a_path.exists() assert a_path.is_absolute() + @pytest.mark.parametrize( 'fmt', Format.__iter__() @@ -71,6 +68,7 @@ def test_format_paths(fmt): """Every format should have an entry in _Path""" assert fmt.name in _Path.items() + def test_no_unmapped_dirs(): """ There should be no additional directories that don't have a mapping for Format. @@ -101,7 +99,7 @@ def test_no_unmapped_dirs(): 'source,fmt', EXPECTED_FORMATS ) -def test_github_path_exists(source,fmt, release_type): +def test_github_path_exists(source, fmt, release_type): url = GITHUB_PATH_FOR(source, fmt, release_type) res = requests.get(url) assert res.status_code != 404, url @@ -115,7 +113,7 @@ def test_github_path_exists(source,fmt, release_type): 'source,fmt', EXPECTED_FORMATS ) -def test_github_path_format(source,fmt, release_type): +def test_github_path_format(source, fmt, release_type): if release_type == ReleaseTag.CURRENT: pytest.skip("Need to cache network requests for this") @@ -125,27 +123,29 @@ def test_github_path_format(source,fmt, release_type): # for windows... assert '\\' not in url + @pytest.mark.skip("github paths largely unused") @pytest.mark.parametrize( 'source,fmt', EXPECTED_FORMATS ) -def test_github_io_path(source,fmt): +def test_github_io_path(source, fmt): url = GITHUB_IO_PATH_FOR(source, fmt) res = requests.get(url) assert res.status_code != 404, url -@pytest.mark.skipif(not HAVE_REQUESTS_CACHE,reason= 'Need to cache this') +@pytest.mark.skipif(not HAVE_REQUESTS_CACHE, reason='Need to cache this') @pytest.mark.parametrize( 'source,fmt', W3ID_FORMATS ) -def test_url_for_format(source,fmt): +def test_url_for_format(source, fmt): url = URL_FOR(source, fmt) res = requests.get(url) assert res.status_code != 404, url + def test_fixed_meta_url(): """ One fixed canary value - the METAMODEL_URI as used in linkml main shouldn't change @@ -153,54 +153,3 @@ def test_fixed_meta_url(): assert URL_FOR(Source.META, Format.YAML) == 'https://w3id.org/linkml/meta.yaml' assert URL_FOR(Source.META, Format.JSONLD) == 'https://w3id.org/linkml/meta.context.jsonld' - -class LinkMLFilesTestCase(unittest.TestCase): - """ Test that linkml_model/linkml_files.py work """ - def test_basic_rules(self): - self.assertEqual("https://w3id.org/linkml/annotations.yaml", - URL_FOR(Source.ANNOTATIONS, Format.YAML)) - self.assertEqual("https://w3id.org/linkml/meta.model.context.jsonld", - URL_FOR(Source.META, Format.NATIVE_JSONLD)) - self.assertEqual(os.path.join(root_path, "model/schema/meta.yaml"), - LOCAL_PATH_FOR(Source.META, Format.YAML)) - print(LOCAL_PATH_FOR(Source.META, Format.YAML)) - self.assertEqual(os.path.join(root_path, "jsonld/types.model.context.jsonld"), - LOCAL_PATH_FOR(Source.TYPES, Format.NATIVE_JSONLD)) - self.assertEqual("https://linkml.github.io/linkml-model/model/schema/meta.yaml", - GITHUB_IO_PATH_FOR(Source.META, Format.YAML)) - self.assertEqual("https://linkml.github.io/linkml-model/jsonld/types.model.context.jsonld", - GITHUB_IO_PATH_FOR(Source.TYPES, Format.NATIVE_JSONLD)) - self.assertEqual("https://raw.githubusercontent.com/linkml/linkml-model/main/jsonld/meta.model.context.jsonld", - GITHUB_PATH_FOR(Source.META, Format.NATIVE_JSONLD, ReleaseTag.LATEST)) - self.assertEqual("https://raw.githubusercontent.com/linkml/linkml-model/testing_branch/owl/mappings.owl.ttl", - GITHUB_PATH_FOR(Source.MAPPINGS, Format.OWL, branch="testing_branch")) - - @unittest.skipIf(SKIP_GITHUB_API, "Github API tests skipped") - def test_github_specific_rules(self): - """ - Test accesses that require github API to access - This is separate because we can only do so many tests per hour w/o getting a 403 - """ - self.assertEqual("https://raw.githubusercontent.com/linkml/linkml-model/f30637f5a585f3fc4b12fd3dbb3e7e95108d4b42/jsonld/meta.model.context.jsonld", - GITHUB_PATH_FOR(Source.META, Format.NATIVE_JSONLD, "v0.0.1")) - current_loc = re.sub(r'linkml-model/[0-9a-f]*/', 'linkml-model/SHA/', GITHUB_PATH_FOR(Source.TYPES, Format.YAML)) - self.assertEqual("https://raw.githubusercontent.com/linkml/linkml-model/SHA/model/schema/types.yaml", current_loc) - # TODO: We may want to raise an error here? - self.assertEqual('https://raw.githubusercontent.com/linkml/linkml-model/missing_branch/owl/mappings.owl.ttl', - GITHUB_PATH_FOR(Source.MAPPINGS, Format.OWL, branch="missing_branch")) - - with self.assertRaises(ValueError) as e: - GITHUB_PATH_FOR(Source.META, Format.RDF, "vv0.0.1") - self.assertEqual("Tag: vv0.0.1 not found!", str(e.exception)) - - def test_shorthand_paths(self): - self.assertEqual('meta', str(fileloc.meta)) - self.assertEqual('meta.yaml', str(fileloc.meta.yaml)) - self.assertEqual('meta.py', str(fileloc.meta.python)) - self.assertEqual(abspath('linkml_model/model/schema/meta.yaml'), str(fileloc.meta.yaml.file)) - self.assertEqual('https://linkml.github.io/linkml-model/model/schema/meta.yaml', str(fileloc.meta.yaml.github_loc())) - self.assertEqual('https://raw.githubusercontent.com/linkml/linkml-model/f30637f5a585f3fc4b12fd3dbb3e7e95108d4b42/model/schema/meta.yaml', str(fileloc.meta.yaml.github_loc('v0.0.1'))) - - -if __name__ == '__main__': - unittest.main() From 1b6598caef604c66b287fec4b4da9caa1b9ee981 Mon Sep 17 00:00:00 2001 From: Sierra Taylor Moxon Date: Mon, 15 Jul 2024 16:57:17 -0700 Subject: [PATCH 3/6] poetry lock --no-update --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index e6b99ee0..0c4e9095 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1938,4 +1938,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "0dad7cd83ffe02a0442c5a4a752c49e63ce651ec13ab29f70655bb4b9bc33ea1" +content-hash = "75087fac676dbc329ed8c48354214e2e768e1aaa37f02c8739ecf1eabdb1c0e3" From d3f07f2a0a591499e851963ef4528b15e8560244 Mon Sep 17 00:00:00 2001 From: Sierra Taylor Moxon Date: Mon, 15 Jul 2024 16:59:14 -0700 Subject: [PATCH 4/6] reduce aggressive new testing of later python versions --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a0a6207f..66d1961d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10"] steps: From 469b58d173a713c665987a9dd0fc04682361ca9e Mon Sep 17 00:00:00 2001 From: Sierra Taylor Moxon Date: Mon, 15 Jul 2024 17:37:57 -0700 Subject: [PATCH 5/6] fixing tests --- .github/workflows/main.yaml | 2 +- linkml_model/linkml_files.py | 175 ++++++++++++++++++----------------- tests/test_linkml_files.py | 8 +- 3 files changed, 94 insertions(+), 91 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 66d1961d..511622d4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11"] steps: diff --git a/linkml_model/linkml_files.py b/linkml_model/linkml_files.py index a8fb89b1..ebd372a9 100644 --- a/linkml_model/linkml_files.py +++ b/linkml_model/linkml_files.py @@ -1,8 +1,8 @@ -import os -from dataclasses import dataclass -from enum import Enum, auto from pathlib import Path -from typing import Optional, Union, Dict +from enum import Enum, auto +from typing import Dict, Optional, Union, NamedTuple +from urllib.parse import urljoin +from dataclasses import dataclass import requests from rdflib import Namespace @@ -11,7 +11,7 @@ LINKML_NAMESPACE = Namespace(LINKML_URL_BASE) GITHUB_IO_BASE = "https://linkml.github.io/linkml-model/" GITHUB_BASE = "https://raw.githubusercontent.com/linkml/linkml-model/" -LOCAL_BASE = os.path.abspath(os.path.dirname(__file__)) +LOCAL_BASE = Path(__file__).parent.resolve() GITHUB_API_BASE = "https://api.github.com/repos/linkml/linkml-model/" GITHUB_RELEASES = GITHUB_BASE + "releases" GITHUB_TAGS = GITHUB_BASE + "tags" @@ -32,90 +32,86 @@ class Source(_AutoName): EXTENSIONS = auto() -class Format(Enum): +class Format(_AutoName): """ LinkML package formats """ - EXCEL = "xlsx" - GRAPHQL = "graphql" - HTML = "" - JSON = "json" - JSONLD = "context.jsonld" - JSON_SCHEMA = "schema.json" - NATIVE_JSONLD = "model.context.jsonld" - NATIVE_RDF = "model.ttl" - NATIVE_SHEXC = "model.shex" - NATIVE_SHEXJ = "model.shexj" - OWL = "owl.ttl" - PREFIXMAP = "yaml" - PROTOBUF = "proto" - PYTHON = "py" - RDF = "ttl" - SHACL = "shacl.ttl" - SHEXC = "shex" - SHEXJ = "shexj" - SQLDDL = "sql" - SQLSCHEMA = "sql" - YAML = "yaml" - - -META_ONLY = ( - Format.EXCEL, - Format.GRAPHQL, - Format.OWL, - Format.PREFIXMAP, - Format.PROTOBUF, - Format.SHACL, - Format.SQLDDL, - Format.SQLSCHEMA -) - + EXCEL = auto() + GRAPHQL = auto() + JSON = auto() + JSONLD = auto() + JSON_SCHEMA = auto() + NATIVE_JSONLD = auto() + NATIVE_RDF = auto() + NATIVE_SHEXC = auto() + NATIVE_SHEXJ = auto() + OWL = auto() + PREFIXMAP = auto() + PROTOBUF = auto() + PYTHON = auto() + RDF = auto() + SHACL = auto() + SHEXC = auto() + SHEXJ = auto() + SQLDDL = auto() + SQLSCHEMA = auto() + YAML = auto() @dataclass class FormatPath: path: str extension: str - def model_path(self, model: str) -> Path: + def model_path(self, model:str) -> Path: return (Path(self.path) / model).with_suffix(self.extension) - class _Path: """ LinkML Relative paths""" - EXCEL = FormatPath("excel", "xlsx") - GRAPHQL = FormatPath("graphql", "graphql") - HTML = FormatPath("", "html") - JSON = FormatPath("json", "json") - JSONLD = FormatPath("jsonld", "context.jsonld") - JSON_SCHEMA = FormatPath("jsonschema", "schema.json") - NATIVE_JSONLD = FormatPath("jsonld", "context.jsonld") - NATIVE_RDF = FormatPath("rdf", "ttl") - NATIVE_SHEXC = FormatPath("shex", "shex") - NATIVE_SHEXJ = FormatPath("shex", "shexj") - OWL = FormatPath("owl", "owl.ttl") - PREFIXMAP = FormatPath('prefixmap', 'yaml') - PROTOBUF = FormatPath("protobuf", "proto") - PYTHON = FormatPath("", "py") - RDF = FormatPath("rdf", "ttl") - SHACL = FormatPath("shacl", "shacl.ttl") - SHEXC = FormatPath("shex", "shex") - SHEXJ = FormatPath("shex", "shexj") - SQLDDL = FormatPath("sqlddl", "sql") - SQLSCHEMA = FormatPath("sqlschema", "sql") - YAML = FormatPath(str(Path("model") / "schema"), "yaml") + EXCEL = FormatPath("excel","xlsx" ) + GRAPHQL = FormatPath("graphql","graphql" ) + JSON = FormatPath("json","json" ) + JSONLD = FormatPath("jsonld","context.jsonld" ) + JSON_SCHEMA = FormatPath("jsonschema", "schema.json" ) + NATIVE_JSONLD = FormatPath("jsonld", "context.jsonld" ) + NATIVE_RDF = FormatPath("rdf","ttl" ) + NATIVE_SHEXC = FormatPath("shex","shex" ) + NATIVE_SHEXJ = FormatPath("shex","shexj" ) + OWL = FormatPath("owl","owl.ttl" ) + PREFIXMAP = FormatPath('prefixmap','yaml' ) + PROTOBUF = FormatPath("protobuf","proto" ) + PYTHON = FormatPath("","py" ) + RDF = FormatPath("rdf","ttl" ) + SHACL = FormatPath("shacl","shacl.ttl" ) + SHEXC = FormatPath("shex","shex" ) + SHEXJ = FormatPath("shex","shexj" ) + SQLDDL = FormatPath("sqlddl","sql" ) + SQLSCHEMA = FormatPath("sqlschema","sql" ) + YAML = FormatPath(str(Path("model") / "schema"),"yaml" ) @classmethod def items(cls) -> Dict[str, FormatPath]: - return {k: v for k, v in cls.__dict__.items() if not k.startswith('_')} + return {k:v for k,v in cls.__dict__.items() if not k.startswith('_')} @classmethod - def get(cls, item: Union[str, Format]) -> FormatPath: + def get(cls, item:Union[str,Format]) -> FormatPath: if isinstance(item, Format): item = item.name.upper() return getattr(cls, item) - def __class_getitem__(cls, item: str) -> FormatPath: + def __class_getitem__(cls, item:str) -> FormatPath: return getattr(cls, item) +META_ONLY = ( + Format.EXCEL, + Format.GRAPHQL, + Format.OWL, + Format.PREFIXMAP, + Format.PROTOBUF, + Format.SHACL, + Format.SQLDDL, + Format.SQLSCHEMA +) + + class ReleaseTag(_AutoName): """ Release tags LATEST - the absolute latest in the supplied branch @@ -124,26 +120,40 @@ class ReleaseTag(_AutoName): CURRENT = auto() -def _build_path(source: Source, fmt: Format) -> str: - """ Create the relative path for source and fmt """ - return f"{_Path[fmt.name].value}/{source.value}.{fmt.value}" +class PathParts(NamedTuple): + format: str + file: str + + +def _build_path(source: Source, fmt: Format) -> PathParts: + """ + Create the parts for a relative path for source and fmt. + Combined elsewhere into a complete path, since OS paths and URLs differ. + """ + fmt_path: FormatPath = _Path.get(fmt.name) + return PathParts(fmt_path.path, f"{source.value}.{fmt_path.extension}") def _build_loc(base: str, source: Source, fmt: Format) -> str: - return f"{base}{_build_path(source, fmt)}".replace('blob/', '') + """A GitHub location""" + # urls are always forward slash separated, so hardcoding is appropriate here + path = '/'.join(_build_path(source, fmt)) + return urljoin(base, path).replace('blob/', '') def URL_FOR(source: Source, fmt: Format) -> str: """ Return the URL to retrieve source in format """ - return f"{LINKML_URL_BASE}{source.value}.{fmt.value}" + fmt_path: FormatPath = _Path.get(fmt.name) + return f"{LINKML_URL_BASE}{source.value}.{fmt_path.extension}" def LOCAL_PATH_FOR(source: Source, fmt: Format) -> str: - return os.path.join(LOCAL_BASE, _build_path(source, fmt)) + return str(LOCAL_BASE.joinpath(*_build_path(source, fmt))) -def GITHUB_IO_PATH_FOR(source: Source, fmt: Format) -> str: - return _build_loc(GITHUB_IO_BASE, source, fmt) +def GITHUB_IO_PATH_FOR(source: Source, fmt: Format, version="latest") -> str: + path = '/'.join([version, 'linkml_model', *_build_path(source, fmt)]) + return urljoin(GITHUB_IO_BASE, path) def GITHUB_PATH_FOR(source: Source, @@ -168,7 +178,8 @@ def tag_to_commit(tag: str) -> str: # Return the absolute latest entry for branch if release is ReleaseTag.LATEST or (release is ReleaseTag.CURRENT and branch != "main"): - return f"{GITHUB_BASE}{branch}/{_build_path(source, fmt)}" + path = '/'.join([branch, 'linkml_model', *_build_path(source, fmt)]) + return urljoin(GITHUB_BASE, path) # Return the latest published version elif release is ReleaseTag.CURRENT: @@ -185,9 +196,10 @@ class ModelLoc: def __init__(self, model: Source, fmt: Format) -> str: self._model = model self._format = fmt + self._fmt_path = _Path.get(fmt.name) def __str__(self): - return f"{self._model.value}.{self._format.value}" + return f"{self._model.value}.{self._fmt_path.extension}" def __repr__(self): return str(self) @@ -200,8 +212,7 @@ def url(self) -> str: def file(self) -> str: return LOCAL_PATH_FOR(self._model, self._format) - def github_loc(self, tag: Optional[str] = None, branch: Optional[str] = None, - release: ReleaseTag = None) -> str: + def github_loc(self, tag: Optional[str] = None, branch: Optional[str] = None, release: ReleaseTag = None) -> str: if not tag and not branch and not release: return GITHUB_IO_PATH_FOR(self._model, self._format) if tag: @@ -218,18 +229,10 @@ def __str__(self): def __repr__(self): return str(self) - @property - def yaml(self) -> ModelLoc: - return ModelFile.ModelLoc(self._model, Format.YAML) - @property def graphql(self) -> ModelLoc: return ModelFile.ModelLoc(self._model, Format.GRAPHQL) - @property - def html(self) -> ModelLoc: - return ModelFile.ModelLoc(self._model, Format.HTML) - @property def json(self) -> ModelLoc: return ModelFile.ModelLoc(self._model, Format.JSON) diff --git a/tests/test_linkml_files.py b/tests/test_linkml_files.py index f05bae3b..91db7f6c 100644 --- a/tests/test_linkml_files.py +++ b/tests/test_linkml_files.py @@ -24,10 +24,10 @@ ReleaseTag ) -EXPECTED_FORMATS = [ - (source, fmt) for source, fmt in product(Source, Format) - if (fmt not in META_ONLY or source == Source.META) -] +EXPECTED_FORMATS = [] +for source, fmt in product(Source, Format): + if fmt not in META_ONLY or source == Source.META: + EXPECTED_FORMATS.append((source, fmt)) W3ID_EXTENSIONS = ( 'html', From e2b031bcee0f1760665cde5f0b71c85a33ce262a Mon Sep 17 00:00:00 2001 From: Sierra Taylor Moxon Date: Mon, 15 Jul 2024 17:59:53 -0700 Subject: [PATCH 6/6] fix tests according to changes in linkml-runtime: for consideration is the change in paths to expect here. noting the 'new' paths do resolve. --- tests/test_linkml_files.py | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_linkml_files.py b/tests/test_linkml_files.py index 91db7f6c..73205894 100644 --- a/tests/test_linkml_files.py +++ b/tests/test_linkml_files.py @@ -1,8 +1,11 @@ +import re + import pytest import requests from pathlib import Path from itertools import product from urllib.parse import urlparse +import os try: import requests_cache @@ -153,3 +156,41 @@ def test_fixed_meta_url(): assert URL_FOR(Source.META, Format.YAML) == 'https://w3id.org/linkml/meta.yaml' assert URL_FOR(Source.META, Format.JSONLD) == 'https://w3id.org/linkml/meta.context.jsonld' + +root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "linkml_model")) + + +def test_basic_rules(): + assert "https://w3id.org/linkml/annotations.yaml" == URL_FOR(Source.ANNOTATIONS, Format.YAML) + assert "https://w3id.org/linkml/meta.context.jsonld" == URL_FOR(Source.META, Format.NATIVE_JSONLD) + assert os.path.join(root_path, "model/schema/meta.yaml") == LOCAL_PATH_FOR(Source.META, Format.YAML) + print(LOCAL_PATH_FOR(Source.META, Format.YAML)) + assert os.path.join(root_path, "jsonld/types.context.jsonld") == LOCAL_PATH_FOR(Source.TYPES, Format.NATIVE_JSONLD) + assert "https://linkml.github.io/linkml-model/latest/linkml_model/model/schema/meta.yaml" == GITHUB_IO_PATH_FOR( + Source.META, Format.YAML) + assert "https://linkml.github.io/linkml-model/latest/linkml_model/jsonld/types.context.jsonld" == \ + GITHUB_IO_PATH_FOR(Source.TYPES, Format.NATIVE_JSONLD) + assert "https://raw.githubusercontent.com/linkml/linkml-model/main/linkml_model/jsonld/meta.context.jsonld" == \ + GITHUB_PATH_FOR(Source.META, Format.NATIVE_JSONLD, ReleaseTag.LATEST) + assert "https://raw.githubusercontent.com/linkml/linkml-model/testing_branch/linkml_model/owl/mappings.owl.ttl" == \ + GITHUB_PATH_FOR(Source.MAPPINGS, Format.OWL, branch="testing_branch") + + +SKIP_GITHUB_API = False + + +@pytest.mark.skipif(SKIP_GITHUB_API, reason="Github API tests skipped") +def test_github_specific_rules(): + """ + Test accesses that require GitHub API to access + This is separate because we can only do so many tests per hour w/o getting a 403 + """ + assert "https://raw.githubusercontent.com/linkml/linkml-model/f30637f5a585f3fc4b12fd3dbb3e7e95108d4b42/jsonld/meta.context.jsonld" == \ + GITHUB_PATH_FOR(Source.META, Format.NATIVE_JSONLD, "v0.0.1") + current_loc = re.sub(r'linkml-model/[0-9a-f]*/', 'linkml-model/SHA/', GITHUB_PATH_FOR(Source.TYPES, Format.YAML)) + assert "https://raw.githubusercontent.com/linkml/linkml-model/SHA/model/schema/types.yaml" == current_loc + assert 'https://raw.githubusercontent.com/linkml/linkml-model/missing_branch/linkml_model/owl/mappings.owl.ttl' == \ + GITHUB_PATH_FOR(Source.MAPPINGS, Format.OWL, branch="missing_branch") + + with pytest.raises(ValueError) as e: + GITHUB_PATH_FOR(Source.META, Format.RDF, "vv0.0.1")