Add DOSDP QC (#1449)

* Adding DOSDP match-QC system

* Update envo.Makefile

.gitignore

@@ -28,3 +28,4 @@ src/envo/envo-full.owl
 src/envo/imports/*_combined_seed.tsv
 /catalog-v001.xml
 src/envo/uberon.obo.1
+src/envo/tmp

src/envo/Makefile

@@ -577,3 +577,5 @@ fill: envo-edit.csv
 
 #modules/%.tsv:
 #	blip-findall -i patterns/spec.obo  -consult pmaker.pro -r envo -r chebi -r pato pattern_$*/3 -no_pred -label -use_tabs > $@
+
+include envo.Makefile

src/envo/envo.Makefile

@@ -0,0 +1,43 @@
+PATTERN_DIR=patterns
+#PATTERN_IDS=$(wildcard ../patterns/dosdp-patterns/[a-z]*.yaml)
+PATTERN_IDS=biome
+ALL_PATTERNS=$(patsubst ../patterns/dosdp-patterns/%.yaml,%,$(PATTERN_IDS))
+DOSDPT=dosdp-tools
+SCRIPTDIR=util
+SPARQLDIR=sparql
+
+.PHONY: matches matches_annotations
+
+tmp/envo-edit-merged.owl: $(SRC)
+	mkdir -p tmp/
+	$(ROBOT) merge -i $< -o $@
+
+PATTERN_MATCH_DIR=patterns/matches
+
+matches: tmp/envo-edit-merged.owl
+	mkdir -p $(PATTERN_MATCH_DIR)/logical
+	$(DOSDPT) query --ontology=$< --catalog=catalog-v001.xml --reasoner=elk --obo-prefixes=true --batch-patterns="$(ALL_PATTERNS)" --template="$(PATTERN_DIR)" --outfile="$(PATTERN_MATCH_DIR)/logical/"
+
+matches_annotations: tmp/envo-edit-merged.owl
+	mkdir -p $(PATTERN_MATCH_DIR)/annotations
+	$(DOSDPT) query --ontology=$< --catalog=catalog-v001.xml --reasoner=elk --restrict-axioms-to=annotation --obo-prefixes=true --batch-patterns="$(ALL_PATTERNS)" --template="$(PATTERN_DIR)" --outfile="$(PATTERN_MATCH_DIR)/annotations/"
+
+##############################################################
+###### Pipeline for adding a compoment with tagging ##########
+# For example: DOSDP conformance
+
+MATCHED_TSVs=$(foreach V,$(notdir $(wildcard $(PATTERN_MATCH_DIR)/logical/*.tsv)),$(PATTERN_MATCH_DIR)/logical/$V)
+
+tmp/tags-dosdp.tsv:
+	python $(SCRIPTDIR)/dosdp-matches-tags.py $(addprefix -d , $(MATCHED_TSVs)) -o $@
+
+tmp/tags-dosdp.owl: tmp/tags-dosdp.tsv
+	$(ROBOT) merge -i $(SRC) template --template $< --output $@
+
+modules/tags.owl: tmp/tags-dosdp.owl
+	$(ROBOT) merge $(addprefix -i , $^) annotate --ontology-iri $(ONTBASE)/$@ -o $@
+
+sparql_test2: $(SRC) modules/tags.owl
+	$(ROBOT) merge $(addprefix -i , $^) verify --queries sparql/dosdp-conforms-violation.sparql -O reports/
+
+test: sparql_test2

src/envo/modules/tags.owl

@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<rdf:RDF xmlns="http://purl.obolibrary.org/obo/envo/modules/tags.owl#"
+     xml:base="http://purl.obolibrary.org/obo/envo/modules/tags.owl"
+     xmlns:owl="http://www.w3.org/2002/07/owl#"
+     xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:xml="http://www.w3.org/XML/1998/namespace"
+     xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
+     xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+     xmlns:terms="http://purl.org/dc/terms/">
+    <owl:Ontology rdf:about="http://purl.obolibrary.org/obo/envo/modules/tags.owl"/>
+
+
+
+    <!-- 
+    ///////////////////////////////////////////////////////////////////////////////////////
+    //
+    // Annotation properties
+    //
+    ///////////////////////////////////////////////////////////////////////////////////////
+     -->
+
+
+
+
+    <!-- http://purl.org/dc/terms/conformsTo -->
+
+    <owl:AnnotationProperty rdf:about="http://purl.org/dc/terms/conformsTo"/>
+
+
+
+    <!-- 
+    ///////////////////////////////////////////////////////////////////////////////////////
+    //
+    // Classes
+    //
+    ///////////////////////////////////////////////////////////////////////////////////////
+     -->
+
+
+
+
+    <!-- http://purl.obolibrary.org/obo/ENVO_00000428 -->
+
+    <owl:Class rdf:about="http://purl.obolibrary.org/obo/ENVO_00000428">
+        <terms:conformsTo rdf:resource="http://purl.obolibrary.org/obo/envo/patterns/biome.yaml"/>
+    </owl:Class>
+</rdf:RDF>
+
+
+
+<!-- Generated by the OWL API (version 4.5.25) https://github.com/owlcs/owlapi -->
+

src/envo/patterns/matches/logical/biome.tsv

@@ -0,0 +1,2 @@
+defined_class	defined_class_label	participant	participant_label
+http://purl.obolibrary.org/obo/ENVO_00000428	biome	http://purl.obolibrary.org/obo/ENVO_01001110	ecosystem

src/envo/reports/dosdp-conforms-violation.csv

@@ -0,0 +1,94 @@
+envo_id,label
+http://purl.obolibrary.org/obo/ENVO_01001838,arid biome
+http://purl.obolibrary.org/obo/ENVO_01001830,tropical biome
+http://purl.obolibrary.org/obo/ENVO_00002030,aquatic biome
+http://purl.obolibrary.org/obo/ENVO_00000873,freshwater biome
+http://purl.obolibrary.org/obo/ENVO_01000253,freshwater river biome
+http://purl.obolibrary.org/obo/ENVO_00000890,small river biome
+http://purl.obolibrary.org/obo/ENVO_00000889,large river delta biome
+http://purl.obolibrary.org/obo/ENVO_00000888,large river headwater biome
+http://purl.obolibrary.org/obo/ENVO_00000887,large river biome
+http://purl.obolibrary.org/obo/ENVO_01000252,freshwater lake biome
+http://purl.obolibrary.org/obo/ENVO_00000892,small freshwater lake biome
+http://purl.obolibrary.org/obo/ENVO_00000891,large freshwater lake biome
+http://purl.obolibrary.org/obo/ENVO_00000893,xeric basin biome
+http://purl.obolibrary.org/obo/ENVO_00000447,marine biome
+http://purl.obolibrary.org/obo/ENVO_01000022,marine salt marsh biome
+http://purl.obolibrary.org/obo/ENVO_01000024,marine benthic biome
+http://purl.obolibrary.org/obo/ENVO_01000027,marine abyssal zone biome
+http://purl.obolibrary.org/obo/ENVO_01000026,marine bathyal zone biome
+http://purl.obolibrary.org/obo/ENVO_01000028,marine hadal zone biome
+http://purl.obolibrary.org/obo/ENVO_01000029,marine reef biome
+http://purl.obolibrary.org/obo/ENVO_01000123,marine sponge reef biome
+http://purl.obolibrary.org/obo/ENVO_01000050,marine subtidal rocky reef biome
+http://purl.obolibrary.org/obo/ENVO_01000049,marine coral reef biome
+http://purl.obolibrary.org/obo/ENVO_01000854,tropical marine coral reef biome
+http://purl.obolibrary.org/obo/ENVO_01000030,marine hydrothermal vent biome
+http://purl.obolibrary.org/obo/ENVO_01000054,marine basaltic hydrothermal vent biome
+http://purl.obolibrary.org/obo/ENVO_01000053,marine ultramafic hydrothermal vent biome
+http://purl.obolibrary.org/obo/ENVO_01000052,marine white smoker biome
+http://purl.obolibrary.org/obo/ENVO_01000051,marine black smoker biome
+http://purl.obolibrary.org/obo/ENVO_01000025,marine neritic benthic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000127,marine cold seep biome
+http://purl.obolibrary.org/obo/ENVO_01000020,estuarine biome
+http://purl.obolibrary.org/obo/ENVO_01000858,marine upwelling biome
+http://purl.obolibrary.org/obo/ENVO_01000860,temperate marine upwelling biome
+http://purl.obolibrary.org/obo/ENVO_01000859,tropical marine upwelling biome
+http://purl.obolibrary.org/obo/ENVO_01000023,marine pelagic biome
+http://purl.obolibrary.org/obo/ENVO_01000033,oceanic pelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000040,oceanic benthopelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000039,oceanic hadal pelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000038,oceanic abyssopelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000037,oceanic bathypelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000036,oceanic mesopelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000035,oceanic epipelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000034,oceanic sea surface microlayer biome
+http://purl.obolibrary.org/obo/ENVO_01000032,neritic pelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000043,neritic mesopelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000042,neritic epipelagic zone biome
+http://purl.obolibrary.org/obo/ENVO_01000041,neritic sea surface microlayer biome
+http://purl.obolibrary.org/obo/ENVO_01000047,mediterranean sea biome
+http://purl.obolibrary.org/obo/ENVO_01000857,temperate mediterranean sea biome
+http://purl.obolibrary.org/obo/ENVO_01000128,dilution basin mediterranean sea biome
+http://purl.obolibrary.org/obo/ENVO_01000004,concentration basin mediterranean sea biome
+http://purl.obolibrary.org/obo/ENVO_01000046,marginal sea biome
+http://purl.obolibrary.org/obo/ENVO_01001230,tropical marginal sea biome
+http://purl.obolibrary.org/obo/ENVO_01000856,temperate marginal sea biome
+http://purl.obolibrary.org/obo/ENVO_01000045,epeiric sea biome
+http://purl.obolibrary.org/obo/ENVO_01001832,subtropical biome
+http://purl.obolibrary.org/obo/ENVO_01001837,subalpine biome
+http://purl.obolibrary.org/obo/ENVO_01001834,subpolar biome
+http://purl.obolibrary.org/obo/ENVO_01001831,temperate biome
+http://purl.obolibrary.org/obo/ENVO_01001836,montane biome
+http://purl.obolibrary.org/obo/ENVO_01000339,polar biome
+http://purl.obolibrary.org/obo/ENVO_00000446,terrestrial biome
+http://purl.obolibrary.org/obo/ENVO_01000176,shrubland biome
+http://purl.obolibrary.org/obo/ENVO_01001369,tidal mangrove shrubland
+http://purl.obolibrary.org/obo/ENVO_01000218,xeric shrubland biome
+http://purl.obolibrary.org/obo/ENVO_01000216,montane shrubland biome
+http://purl.obolibrary.org/obo/ENVO_01000215,temperate shrubland biome
+http://purl.obolibrary.org/obo/ENVO_01000214,tropical shrubland biome
+http://purl.obolibrary.org/obo/ENVO_01000213,subtropical shrubland biome
+http://purl.obolibrary.org/obo/ENVO_01000217,mediterranean shrubland biome
+http://purl.obolibrary.org/obo/ENVO_01000180,tundra biome
+http://purl.obolibrary.org/obo/ENVO_01001505,alpine tundra biome
+http://purl.obolibrary.org/obo/ENVO_01000175,woodland biome
+http://purl.obolibrary.org/obo/ENVO_01000222,subtropical woodland biome
+http://purl.obolibrary.org/obo/ENVO_01000208,mediterranean woodland biome
+http://purl.obolibrary.org/obo/ENVO_01000221,temperate woodland biome
+http://purl.obolibrary.org/obo/ENVO_01000220,tropical woodland biome
+http://purl.obolibrary.org/obo/ENVO_01000178,savanna biome
+http://purl.obolibrary.org/obo/ENVO_01000223,montane savanna biome
+http://purl.obolibrary.org/obo/ENVO_01000190,flooded savanna biome
+http://purl.obolibrary.org/obo/ENVO_01000189,temperate savanna biome
+http://purl.obolibrary.org/obo/ENVO_01000188,tropical savanna biome
+http://purl.obolibrary.org/obo/ENVO_01000187,subtropical savanna biome
+http://purl.obolibrary.org/obo/ENVO_01000229,mediterranean savanna biome
+http://purl.obolibrary.org/obo/ENVO_01000181,mangrove biome
+http://purl.obolibrary.org/obo/ENVO_01000219,anthropogenic terrestrial biome
+http://purl.obolibrary.org/obo/ENVO_01000248,dense settlement biome
+http://purl.obolibrary.org/obo/ENVO_01000249,urban biome
+http://purl.obolibrary.org/obo/ENVO_01000247,rangeland biome
+http://purl.obolibrary.org/obo/ENVO_01000246,village biome
+http://purl.obolibrary.org/obo/ENVO_01001833,mediterranean biome
+http://purl.obolibrary.org/obo/ENVO_01001835,alpine biome

src/envo/sparql/dosdp-conforms-violation.sparql

@@ -0,0 +1,12 @@
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+prefix MONDO: <http://purl.obolibrary.org/obo/MONDO_>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+prefix oio: <http://www.geneontology.org/formats/oboInOwl#>
+
+SELECT ?envo_id ?label WHERE {
+  ?envo_id rdfs:subClassOf+ <http://purl.obolibrary.org/obo/ENVO_00000428> .
+  ?envo_id rdfs:label ?label .
+  FILTER NOT EXISTS {
+  	?envo_id <http://purl.org/dc/terms/conformsTo> <http://purl.obolibrary.org/obo/envo/patterns/biome.yaml> . 
+  }
+}

src/envo/util/dosdp-matches-tags.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+"""
+author: Nico Matentzoglu, 26 August 2020
+"""
+
+import pandas as pd
+from pathlib import Path
+from argparse import ArgumentParser
+
+patterns_iri="http://purl.obolibrary.org/obo/envo/patterns/"
+
+def get_dataframe(tsv):
+    try:
+        df = pd.read_csv(tsv,sep="\t")
+        df["pattern"]=patterns_iri+Path(tsv).stem+".yaml"
+        return df[["defined_class","pattern"]]
+    except pd.errors.EmptyDataError:
+        print("WARNING! ", tsv, " is empty and has been skipped.")
+    return pd.DataFrame(columns=['defined_class','pattern'])
+
+parser = ArgumentParser()
+parser.add_argument(
+    "-d",
+    "--dosdp-tsv",
+    action="append",
+    dest="dosdp_tsvs",
+    help="The set of mapping suggestions to be merged.",
+)
+parser.add_argument("-o", "--output", dest="tsv_out_path", help="Output file", metavar="FILE")
+args = parser.parse_args()
+
+merge = []
+merge.append(pd.DataFrame({'defined_class':  ['ID'], 'pattern': ['AI dc:conformsTo']},columns=['defined_class','pattern']))
+merge.extend([get_dataframe(f) for f in args.dosdp_tsvs])
+
+df = pd.concat(merge)
+df.reset_index(drop=True, inplace=True)
+
+print(df.head())
+
+df.to_csv(args.tsv_out_path,sep="\t",index=False)
+