diff --git a/.travis.yml b/.travis.yml index 41b003f7f..9de1f2238 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,8 @@ before_deploy: - mkdir -p dist - cp target/drafter.jar dist/drafter-$TRAVIS_BRANCH-latest.jar - cp target/drafter.jar dist/drafter-private-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER.jar +- mkdir releases +- cp target/drafter.jar releases/drafter-$TRAVIS_TAG-$TRAVIS_BUILD_NUMBER.jar deploy: - provider: s3 access_key_id: AKIAJZVFELTIBDDYV5OQ @@ -40,3 +42,18 @@ deploy: on: repo: Swirrl/drafter branch: master + - provider: releases + api_key: "$RELEASE_OAUTH_TOKEN" + file: "releases/drafter-$TRAVIS_TAG-$TRAVIS_BUILD_NUMBER.jar" + skip_cleanup: true + on: + tags: true + condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ + - provider: releases + api_key: "$RELEASE_OAUTH_TOKEN" + file: "releases/drafter-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER.jar" + prerelease: true + skip_cleanup: true + on: + tags: true + condition: $TRAVIS_TAG =~ ^[a-z,A-Z]+.* diff --git a/README.md b/README.md index 75a3e08d8..8c8760ca6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ A RESTful Clojure web service to support PMD's admin tool in moving data updates between draft and live triple stores. -Be sure to see the [Getting Started Guide](https://github.com/Swirrl/drafter/blob/master/doc/getting-started.org) for how to use Drafter and set up your Dev environment. +- Using Drafter as part of PMD or otherwise? Then see the [User Guide](https://github.com/Swirrl/drafter/blob/master/doc/using-drafter.md) +- Developing on Drafter itself? Then see the [Getting Started Guide](https://github.com/Swirrl/drafter/blob/master/doc/getting-started.org) for how to use Drafter and set up your Dev environment. ## Configuring Drafter diff --git a/doc/using-drafter.md b/doc/using-drafter.md new file mode 100644 index 000000000..5e73d2e61 --- /dev/null +++ b/doc/using-drafter.md @@ -0,0 +1,23 @@ +# Using Drafter + +`TODO` Make this more of a guide on how to run drafter as a user etc. + +## Convert any existing Grafts to work with Drafter + +If you have an existing Database with quads in it you may need to make drafter aware of them. To do this you can run the update statement though be careful not to run it twice on the same database! + +```sparql +INSERT { + GRAPH { + ?g a . + ?g true . + } +} WHERE { + SELECT DISTINCT ?g + WHERE { + GRAPH ?g { + ?s ?p ?o . + } + } +} +``` diff --git a/env/dev/resources/test-queries/empty-values.sparql b/env/dev/resources/test-queries/empty-values.sparql new file mode 100644 index 000000000..63118153c --- /dev/null +++ b/env/dev/resources/test-queries/empty-values.sparql @@ -0,0 +1,7 @@ +SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds a + } +} diff --git a/project.clj b/project.clj index 3307bb468..ef369225c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,6 @@ -(defproject drafter "2.1.9-SNAPSHOT" +(def VERSION (or (System/getenv "TRAVIS_TAG") "local-2.1.x-SNAPSHOT")) + +(defproject drafter VERSION :description "Backend PMD service" :url "http://github.com/Swirrl/drafter" :license {:name "Proprietary & Commercially Licensed Only" @@ -15,6 +17,11 @@ ;; :passphrase :env ;; :snapshots false}]] + :repositories [["apache-dev" {:url "https://repository.apache.org/content/repositories/snapshots/" + :releases false}] + + ] + :classifiers {:prod :prod :dev :dev} @@ -63,13 +70,20 @@ [metosin/ring-swagger-ui "2.2.10"] ;; Use JENA for our query rewriting - [org.apache.jena/jena-arq "3.4.0" :exclusions [org.slf4j/slf4j-api - org.slf4j/jcl-over-slf4j - org.apache.httpcomponents/httpclient]] - [org.apache.jena/jena-base "3.4.0" :exclusions [org.slf4j/slf4j-api]] - [org.apache.jena/jena-core "3.4.0" :exclusions [org.slf4j/slf4j-api]] - [org.apache.jena/jena-iri "3.4.0" :exclusions [org.slf4j/slf4j-api]] + + + ;; Lock JENA to a specific 3.7.0-SNAPSHOT based on this commit: + ;; https://github.com/afs/jena/tree/2586abf09751fd962ca7afe25d4ab3d9431ce716 + ;; + ;; We should update these deps when 3.7.0 is released. + [org.apache.jena/jena-arq "3.7.0-20180131.100034-17" :exclusions [org.slf4j/slf4j-api + org.slf4j/jcl-over-slf4j + org.apache.httpcomponents/httpclient]] + [org.apache.jena/jena-base "3.7.0-20180131.095758-17" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-core "3.7.0-20180131.095918-17" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-iri "3.7.0-20180131.095726-17" :exclusions [org.slf4j/slf4j-api]] + [org.mindrot/jbcrypt "0.4"] [org.slf4j/slf4j-log4j12 "1.7.25" :exclusions [log4j org.slf4j/slf4j-api]] diff --git a/src/drafter/backend/draftset/arq.clj b/src/drafter/backend/draftset/arq.clj index c7fc7661f..e50f18f6a 100644 --- a/src/drafter/backend/draftset/arq.clj +++ b/src/drafter/backend/draftset/arq.clj @@ -67,7 +67,7 @@ (.isList i))) (defn sse-zipper - "Construct a zipper on a Sparql SExpression Item, that allows easy persistent + "Construct a zipper on a Sparql SExpression Item, that allows easy walking of the sparql algebra tree." [sse-item] (z/zipper sse-list-item? @@ -104,15 +104,36 @@ in the SSE algebra tree, such as the set of prefixes and the query type." [rewriter qstr] - (let [q (sparql-string->arq-query qstr) - transform-query (fn [q] - (-> q - ->sse-item - sse-zipper - rewriter - str - (SSE/parseOp (.getPrefixMapping q)) - OpAsQuery/asQuery))] + (let [q (QueryFactory/create qstr Syntax/syntaxSPARQL_11) + ;q (sparql-string->arq-query qstr) + transform-query (fn [q] + (-> q + ->sse-item + sse-zipper + rewriter + str + (SSE/parseOp (.getPrefixMapping q)) + OpAsQuery/asQuery) + + #_(-> q + ->sse-item + sse-zipper + rewriter + + + org.apache.jena.sparql.sse.builders.BuilderOp/build + + + OpAsQuery/asQuery) + + #_(->> (-> q + + Algebra/compile + sse-zipper) + + rewriter + org.apache.jena.sparql.sse.builders.BuilderOp/build + OpAsQuery/asQuery))] (doto (cond ;; NOTE that to support primitive DESCRIBE queries such as those of @@ -155,6 +176,14 @@ (comment + (println (str (->sse-item (sparql-string->arq-query "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +}")))) + ;; Playing with zippers (let [item (->sse-item (sparql-string->arq-query "SELECT * WHERE { GRAPH { ?s ?p ?o } GRAPH { ?s ?p ?o } }"))] @@ -174,34 +203,119 @@ SSE/parseOp OpAsQuery/asQuery)) -;; constructs are different... -(str ( (query->rewritable "PREFIX : CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") identity)) + ;; constructs are different... + (str ( (query->rewritable "PREFIX : CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") identity)) -;; NOTE it returns a SELECT... weird... I think it's because select/construct/ask is decided outside the algebra -;; => "PREFIX : \n\nSELECT *\nWHERE\n { ?s ?p ?o .\n ?s2 ?p ?o2\n }\n" + ;; NOTE it returns a SELECT... weird... I think it's because select/construct/ask is decided outside the algebra + ;; => "PREFIX : \n\nSELECT *\nWHERE\n { ?s ?p ?o .\n ?s2 ?p ?o2\n }\n" -;; https://jena.apache.org/documentation/query/algebra.html + ;; https://jena.apache.org/documentation/query/algebra.html -;; round tripping from sparql string -> sse algebra -> sparql string -(-> (QueryFactory/create "PREFIX dcterms: SELECT * WHERE { ?s ?p ?o }") - Algebra/compile - OpAsQuery/asQuery) + ;; round tripping from sparql string -> sse algebra -> sparql string + (-> (QueryFactory/create "PREFIX dcterms: SELECT * WHERE { ?s ?p ?o }") + Algebra/compile + OpAsQuery/asQuery) -(-> (QueryFactory/create "PREFIX dcterms: SELECT * WHERE { ?s ?p ?o }") - Algebra/compile - OpAsQuery/asQuery - str - SSE/parseItem - .getList - seq) + (-> (QueryFactory/create "PREFIX dcterms: SELECT * WHERE { ?s ?p ?o }") + Algebra/compile + OpAsQuery/asQuery + str + SSE/parseItem + .getList + seq) -(QueryFactory/create "PREFIX : CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") + (QueryFactory/create "PREFIX : CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") -(Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")) + (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")) -(Algebra/optimize (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }"))) + (Algebra/optimize (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }"))) -(Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")) + + (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")) -(last (.getList (SSE/parseItem "(graph :foo (bgp (triple ?s ?p ?o)))"))) -) + (last (.getList (SSE/parseItem "(graph :foo (bgp (triple ?s ?p ?o)))"))) + + + + ;; (OpAsQuery/asQuery (drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {} (Algebra/compile (sparql-string->arq-query "SELECT DISTINCT ?mdg + ;; WHERE { + ;; VALUES ?mdg { } + ;; GRAPH ?mdg { + ;; ?ds . + ;; } + ;; } + ;; ")))) + + ;; simplified + (->> (-> + "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +} +" + sparql-string->arq-query + Algebra/compile + sse-zipper + ) + (drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {} ) + OpAsQuery/asQuery) + + + (def mdg-query "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +} +") + + ;; expanded + (->> (-> + mdg-query + (QueryFactory/create Syntax/syntaxSPARQL_11) + Algebra/compile + sse-zipper + ) + + (drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {} ) + OpAsQuery/asQuery) + ;; above works wtf... + + (-> mdg-query + sparql-string->arq-query + ->sse-item + sse-zipper + z/root + ;#_str + ;#_(SSE/parseOp (.getPrefixMapping q)) + ;#_OpAsQuery/asQuery + + + org.apache.jena.sparql.sse.builders.BuilderOp/build + + OpAsQuery/asQuery) + + + (-> + (partial drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {}) + (apply-rewriter "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +} +") + str) + + (println "SELECT DISTINCT ?mdg\nWHERE\n { VALUES ?mdg { }\n GRAPH ?mdg\n { ?ds a }\n }\n") + + + + (OpAsQuery/asQuery (Algebra/compile (QueryFactory/create "SELECT ?foo WHERE { ?foo ?bar ?baz . VALUES ?foo {} }"))) + + ) diff --git a/src/drafter/backend/draftset/rewrite_query.clj b/src/drafter/backend/draftset/rewrite_query.clj index 123bef0b2..c15d26fc7 100644 --- a/src/drafter/backend/draftset/rewrite_query.clj +++ b/src/drafter/backend/draftset/rewrite_query.clj @@ -57,5 +57,14 @@ (apply-rewriter (partial uri-constant-rewriter {"http://foo.com/" "http://bar.com/"}) "SELECT * WHERE { GRAPH { ?s ?p ?o }}") + (apply-rewriter (partial uri-constant-rewriter {}) + "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +}") + ) diff --git a/src/drafter/feature/draftset/create.clj b/src/drafter/feature/draftset/create.clj index ced257fb6..f3c9f5fac 100644 --- a/src/drafter/feature/draftset/create.clj +++ b/src/drafter/feature/draftset/create.clj @@ -1,5 +1,7 @@ (ns drafter.feature.draftset.create (:require [drafter.backend.draftset.operations :as dsops] + [clojure.spec.alpha :as s] + [integrant.core :as ig] [drafter.feature.common :as feat-common] [drafter.rdf.draftset-management.job-util :as jobutil] [drafter.util :as util] diff --git a/test/drafter/backend/draftset/arq_test.clj b/test/drafter/backend/draftset/arq_test.clj index 6c0aa71de..57db28737 100644 --- a/test/drafter/backend/draftset/arq_test.clj +++ b/test/drafter/backend/draftset/arq_test.clj @@ -2,6 +2,7 @@ "Testing round tripping through Jena ARC" (:require [clojure.java.io :as io] [clojure.test :as t] + [clojure.string :as str] [drafter.backend.draftset.arq :as sut])) (defn load-query [res-path] @@ -33,4 +34,17 @@ "Round tripping twice generates same query string"))) (t/deftest rewriting-with-sparql-group-concats - (t/is (roundtripable? (load-query "test-queries/group-concat.sparql")))) + (t/is (roundtripable? (load-query "test-queries/group-concat.sparql"))) + (t/is (roundtripable? (load-query "test-queries/empty-values.sparql")))) + +(defn normalise-whitespace + "Replace all consecutive whitespace with a single space." + [q] + (str/replace q #"\s+" " ")) + +(t/deftest preserves-empty-values-blocks + (let [normalised-query (normalise-whitespace (load-query "test-queries/empty-values.sparql"))] + (t/is (= normalised-query (normalise-whitespace (round-trip-query-string normalised-query))) + "Empty Values blocks are preserved"))) + +