Merge remote-tracking branch 'origin/master' into stasher-feature-reorg

Swirrl · Jan 31, 2018 · b47b2b2 · b47b2b2
2 parents 5d491f1 + ed1e65b
commit b47b2b2
Show file tree

Hide file tree

Showing 9 changed files with 242 additions and 41 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -27,6 +27,8 @@ before_deploy:
 - mkdir -p dist
 - cp target/drafter.jar dist/drafter-$TRAVIS_BRANCH-latest.jar
 - cp target/drafter.jar dist/drafter-private-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER.jar
+- mkdir releases
+- cp target/drafter.jar releases/drafter-$TRAVIS_TAG-$TRAVIS_BUILD_NUMBER.jar
 deploy:
   - provider: s3
     access_key_id: AKIAJZVFELTIBDDYV5OQ
@@ -40,3 +42,18 @@ deploy:
     on:
       repo: Swirrl/drafter
       branch: master
+  - provider: releases
+    api_key: "$RELEASE_OAUTH_TOKEN"
+    file: "releases/drafter-$TRAVIS_TAG-$TRAVIS_BUILD_NUMBER.jar"
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$
+  - provider: releases
+    api_key: "$RELEASE_OAUTH_TOKEN"
+    file: "releases/drafter-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER.jar"
+    prerelease: true
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: $TRAVIS_TAG =~ ^[a-z,A-Z]+.*
diff --git a/README.md b/README.md
@@ -4,7 +4,8 @@
 
 A RESTful Clojure web service to support PMD's admin tool in moving data updates between draft and live triple stores.
 
-Be sure to see the [Getting Started Guide](https://github.com/Swirrl/drafter/blob/master/doc/getting-started.org) for how to use Drafter and set up your Dev environment.
+- Using Drafter as part of PMD or otherwise?  Then see the [User Guide](https://github.com/Swirrl/drafter/blob/master/doc/using-drafter.md)
+- Developing on Drafter itself?  Then see the [Getting Started Guide](https://github.com/Swirrl/drafter/blob/master/doc/getting-started.org) for how to use Drafter and set up your Dev environment.
 
 ## Configuring Drafter
 

diff --git a/doc/using-drafter.md b/doc/using-drafter.md
@@ -0,0 +1,23 @@
+# Using Drafter
+
+`TODO` Make this more of a guide on how to run drafter as a user etc.
+
+## Convert any existing Grafts to work with Drafter
+
+If you have an existing Database with quads in it you may need to make drafter aware of them.  To do this you can run the update statement though be careful not to run it twice on the same database!
+
+```sparql
+INSERT {
+  GRAPH <http://publishmydata.com/graphs/drafter/drafts> {
+    ?g a <http://publishmydata.com/def/drafter/ManagedGraph> .
+    ?g <http://publishmydata.com/def/drafter/isPublic> true .
+  }
+} WHERE {
+  SELECT DISTINCT ?g
+  WHERE {
+    GRAPH ?g {
+       ?s ?p ?o .
+    }
+  }
+}
+```
diff --git a/env/dev/resources/test-queries/empty-values.sparql b/env/dev/resources/test-queries/empty-values.sparql
@@ -0,0 +1,7 @@
+SELECT DISTINCT ?mdg 
+WHERE  {
+  VALUES ?mdg {  }
+         GRAPH ?mdg {
+           ?ds a <http://publishmydata.com/def/dataset#Dataset> 
+         }
+}
diff --git a/project.clj b/project.clj
@@ -1,4 +1,6 @@
-(defproject drafter "2.1.9-SNAPSHOT"
+(def VERSION (or (System/getenv "TRAVIS_TAG") "local-2.1.x-SNAPSHOT"))
+
+(defproject drafter VERSION
   :description "Backend PMD service"
   :url "http://github.com/Swirrl/drafter"
   :license {:name "Proprietary & Commercially Licensed Only"
@@ -15,6 +17,11 @@
   ;;                             :passphrase :env
   ;;                             :snapshots false}]]
 
+  :repositories [["apache-dev" {:url "https://repository.apache.org/content/repositories/snapshots/"
+                                :releases false}]
+
+                 ]
+
   :classifiers {:prod :prod
                 :dev :dev}
 
@@ -63,13 +70,20 @@
                  [metosin/ring-swagger-ui "2.2.10"]
 
                  ;; Use JENA for our query rewriting
-                 [org.apache.jena/jena-arq "3.4.0" :exclusions [org.slf4j/slf4j-api
-                                                                org.slf4j/jcl-over-slf4j
-                                                                org.apache.httpcomponents/httpclient]]
-                 [org.apache.jena/jena-base "3.4.0" :exclusions [org.slf4j/slf4j-api]]
-                 [org.apache.jena/jena-core "3.4.0" :exclusions [org.slf4j/slf4j-api]]
-                 [org.apache.jena/jena-iri "3.4.0" :exclusions [org.slf4j/slf4j-api]]
+
+
+                 ;; Lock JENA to a specific 3.7.0-SNAPSHOT based on this commit: 
+                 ;; https://github.com/afs/jena/tree/2586abf09751fd962ca7afe25d4ab3d9431ce716
+                 ;;
+                 ;; We should update these deps when 3.7.0 is released.
+                 [org.apache.jena/jena-arq "3.7.0-20180131.100034-17" :exclusions [org.slf4j/slf4j-api
+                                                                                   org.slf4j/jcl-over-slf4j
+                                                                                   org.apache.httpcomponents/httpclient]]
+                 [org.apache.jena/jena-base "3.7.0-20180131.095758-17" :exclusions [org.slf4j/slf4j-api]]
+                 [org.apache.jena/jena-core "3.7.0-20180131.095918-17" :exclusions [org.slf4j/slf4j-api]]
+                 [org.apache.jena/jena-iri "3.7.0-20180131.095726-17" :exclusions [org.slf4j/slf4j-api]]
 
+
                  [org.mindrot/jbcrypt "0.4"]
 
                  [org.slf4j/slf4j-log4j12 "1.7.25" :exclusions [log4j org.slf4j/slf4j-api]]

diff --git a/src/drafter/backend/draftset/arq.clj b/src/drafter/backend/draftset/arq.clj
@@ -67,7 +67,7 @@
        (.isList i)))
 
 (defn sse-zipper
-  "Construct a zipper on a Sparql SExpression Item, that allows easy persistent
+  "Construct a zipper on a Sparql SExpression Item, that allows easy 
   walking of the sparql algebra tree."
   [sse-item]
   (z/zipper sse-list-item?
@@ -104,15 +104,36 @@
   in the SSE algebra tree, such as the set of prefixes and the query type."
   [rewriter qstr]
 
-  (let [q (sparql-string->arq-query qstr)
-        transform-query (fn [q]
-                          (-> q
-                              ->sse-item
-                              sse-zipper
-                              rewriter
-                              str
-                              (SSE/parseOp (.getPrefixMapping q))
-                              OpAsQuery/asQuery))]
+  (let [q (QueryFactory/create qstr Syntax/syntaxSPARQL_11)
+        ;q (sparql-string->arq-query qstr)
+         transform-query (fn [q]
+                           (-> q
+                               ->sse-item
+                               sse-zipper
+                               rewriter
+                               str
+                               (SSE/parseOp (.getPrefixMapping q))
+                               OpAsQuery/asQuery)
+
+                           #_(-> q
+                               ->sse-item
+                               sse-zipper
+                               rewriter
+
+
+                               org.apache.jena.sparql.sse.builders.BuilderOp/build
+
+
+                             OpAsQuery/asQuery)
+
+                           #_(->> (-> q
+
+                                    Algebra/compile
+                                    sse-zipper)
+
+                                rewriter
+                                org.apache.jena.sparql.sse.builders.BuilderOp/build
+                                OpAsQuery/asQuery))]
 
     (doto (cond
             ;; NOTE that to support primitive DESCRIBE queries such as those of
@@ -155,6 +176,14 @@
 
 (comment
 
+  (println (str (->sse-item (sparql-string->arq-query "SELECT DISTINCT ?mdg
+WHERE  {
+ VALUES ?mdg { <urn:val> }
+         GRAPH ?mdg {
+           ?ds <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://publishmydata.com/def/dataset#Dataset> .
+         }
+}"))))
+
   ;; Playing with zippers
 
   (let [item (->sse-item (sparql-string->arq-query "SELECT * WHERE { GRAPH <http://replace-me.com> { ?s ?p ?o } GRAPH <http://bar.com> { ?s ?p ?o } }"))]
@@ -174,34 +203,119 @@
         SSE/parseOp
         OpAsQuery/asQuery))
 
-;; constructs are different...
-(str ( (query->rewritable "PREFIX : <http://foo> CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") identity))
+  ;; constructs are different...
+  (str ( (query->rewritable "PREFIX : <http://foo> CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") identity))
 
-;; NOTE it returns a SELECT... weird... I think it's because select/construct/ask is decided outside the algebra
-;; => "PREFIX  :     <http://foo>\n\nSELECT  *\nWHERE\n  { ?s ?p ?o .\n    ?s2 ?p ?o2\n  }\n"
+  ;; NOTE it returns a SELECT... weird... I think it's because select/construct/ask is decided outside the algebra
+  ;; => "PREFIX  :     <http://foo>\n\nSELECT  *\nWHERE\n  { ?s ?p ?o .\n    ?s2 ?p ?o2\n  }\n"
 
-;; https://jena.apache.org/documentation/query/algebra.html
+  ;; https://jena.apache.org/documentation/query/algebra.html
 
-;; round tripping from sparql string -> sse algebra -> sparql string
-(-> (QueryFactory/create "PREFIX dcterms: <http://purl.org/dc/terms/> SELECT * WHERE { ?s ?p ?o }")
-    Algebra/compile
-    OpAsQuery/asQuery)
+  ;; round tripping from sparql string -> sse algebra -> sparql string
+  (-> (QueryFactory/create "PREFIX dcterms: <http://purl.org/dc/terms/> SELECT * WHERE { ?s ?p ?o }")
+      Algebra/compile
+      OpAsQuery/asQuery)
 
-(-> (QueryFactory/create "PREFIX dcterms: <http://purl.org/dc/terms/>   SELECT * WHERE { ?s ?p ?o }")
-    Algebra/compile
-    OpAsQuery/asQuery
-    str
-    SSE/parseItem
-    .getList
-    seq)
+  (-> (QueryFactory/create "PREFIX dcterms: <http://purl.org/dc/terms/>   SELECT * WHERE { ?s ?p ?o }")
+      Algebra/compile
+      OpAsQuery/asQuery
+      str
+      SSE/parseItem
+      .getList
+      seq)
 
-(QueryFactory/create "PREFIX : <http://foo> CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }")
+  (QueryFactory/create "PREFIX : <http://foo> CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }")
 
-(Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }"))
+  (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }"))
 
-(Algebra/optimize (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")))
+  (Algebra/optimize (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")))
 
-(Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }"))
+
+  (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }"))
 
-(last (.getList (SSE/parseItem "(graph :foo (bgp (triple ?s ?p ?o)))")))
-)
+  (last (.getList (SSE/parseItem "(graph :foo (bgp (triple ?s ?p ?o)))")))
+
+
+
+  ;; (OpAsQuery/asQuery (drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {} (Algebra/compile (sparql-string->arq-query "SELECT DISTINCT ?mdg 
+  ;; WHERE  {
+  ;;   VALUES ?mdg {  }
+  ;;          GRAPH ?mdg {
+  ;;            ?ds <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://publishmydata.com/def/dataset#Dataset> .
+  ;;          }
+  ;; }
+  ;; "))))
+
+  ;; simplified
+  (->> (-> 
+        "SELECT DISTINCT ?mdg
+WHERE  {
+  VALUES ?mdg {  }
+         GRAPH ?mdg {
+           ?ds <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://publishmydata.com/def/dataset#Dataset> .
+         }
+}
+"
+        sparql-string->arq-query
+        Algebra/compile
+        sse-zipper
+        )
+       (drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {} )
+       OpAsQuery/asQuery)
+
+
+  (def mdg-query "SELECT DISTINCT ?mdg
+WHERE  {
+  VALUES ?mdg { }
+         GRAPH ?mdg {
+           ?ds <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://publishmydata.com/def/dataset#Dataset> .
+         }
+}
+")
+
+  ;; expanded
+  (->> (-> 
+        mdg-query
+        (QueryFactory/create Syntax/syntaxSPARQL_11)
+        Algebra/compile
+        sse-zipper
+        )
+
+       (drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {} )
+       OpAsQuery/asQuery)
+  ;; above works wtf...
+
+  (-> mdg-query
+      sparql-string->arq-query
+      ->sse-item
+      sse-zipper
+      z/root
+                                        ;#_str
+                                        ;#_(SSE/parseOp (.getPrefixMapping q))
+                                        ;#_OpAsQuery/asQuery
+
+
+      org.apache.jena.sparql.sse.builders.BuilderOp/build
+
+      OpAsQuery/asQuery)
+
+
+  (-> 
+      (partial drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {})
+      (apply-rewriter "SELECT DISTINCT ?mdg
+WHERE  {
+  VALUES ?mdg { }
+         GRAPH ?mdg {
+           ?ds <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://publishmydata.com/def/dataset#Dataset> .
+         }
+}
+")
+      str)
+
+  (println "SELECT DISTINCT  ?mdg\nWHERE\n  { VALUES ?mdg { }\n    GRAPH ?mdg\n      { ?ds  a                     <http://publishmydata.com/def/dataset#Dataset> }\n  }\n")
+
+
+
+  (OpAsQuery/asQuery (Algebra/compile (QueryFactory/create "SELECT ?foo WHERE { ?foo ?bar ?baz . VALUES ?foo {} }")))
+
+  )
diff --git a/src/drafter/backend/draftset/rewrite_query.clj b/src/drafter/backend/draftset/rewrite_query.clj
@@ -57,5 +57,14 @@
   (apply-rewriter (partial uri-constant-rewriter {"http://foo.com/" "http://bar.com/"})
                   "SELECT * WHERE { GRAPH <http://foo.com/> { ?s ?p ?o }}")
 
+  (apply-rewriter (partial uri-constant-rewriter {})
+                  "SELECT DISTINCT ?mdg 
+WHERE  {
+  VALUES ?mdg {  }
+         GRAPH ?mdg {
+           ?ds <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://publishmydata.com/def/dataset#Dataset> .
+         }
+}")
+
 
 )
diff --git a/src/drafter/feature/draftset/create.clj b/src/drafter/feature/draftset/create.clj
@@ -1,5 +1,7 @@
 (ns drafter.feature.draftset.create
   (:require [drafter.backend.draftset.operations :as dsops]
+            [clojure.spec.alpha :as s]
+            [integrant.core :as ig]
             [drafter.feature.common :as feat-common]
             [drafter.rdf.draftset-management.job-util :as jobutil]
             [drafter.util :as util]

diff --git a/test/drafter/backend/draftset/arq_test.clj b/test/drafter/backend/draftset/arq_test.clj
@@ -2,6 +2,7 @@
   "Testing round tripping through Jena ARC"
   (:require [clojure.java.io :as io]
             [clojure.test :as t]
+            [clojure.string :as str]
             [drafter.backend.draftset.arq :as sut]))
 
 (defn load-query [res-path]
@@ -33,4 +34,17 @@
           "Round tripping twice generates same query string")))
 
 (t/deftest rewriting-with-sparql-group-concats
-  (t/is (roundtripable? (load-query "test-queries/group-concat.sparql"))))
+  (t/is (roundtripable? (load-query "test-queries/group-concat.sparql")))
+  (t/is (roundtripable? (load-query "test-queries/empty-values.sparql"))))
+
+(defn normalise-whitespace
+  "Replace all consecutive whitespace with a single space."
+  [q]
+  (str/replace q #"\s+" " "))
+
+(t/deftest preserves-empty-values-blocks
+  (let [normalised-query (normalise-whitespace (load-query "test-queries/empty-values.sparql"))]
+    (t/is (= normalised-query (normalise-whitespace (round-trip-query-string normalised-query)))
+          "Empty Values blocks are preserved")))
+
+