From 0eea6f12ed5868ece396bc133a45b792518f4fbd Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 25 Oct 2017 12:51:13 +0100 Subject: [PATCH 01/16] bump grafter dependency to fix rdf4j registry update issue --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index d236187d0..bad3dc334 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject drafter "2.1.7-SNAPSHOT" +(defproject drafter "2.1.8-SNAPSHOT" :description "Backend PMD service" :url "http://github.com/Swirrl/drafter" :license {:name "Proprietary & Commercially Licensed Only" @@ -45,7 +45,7 @@ [com.sun.mail/javax.mail "1.6.0"] ;;[com.taoensso/tower "2.0.2"] - [grafter "0.10.0"] + [grafter "0.10.1"] [grafter/url "0.2.5"] ;[grafter/vocabularies "0.1.3"] [lib-noir "0.9.9" :exclusions [compojure org.clojure/java.classpath org.clojure/tools.reader org.clojure/java.classpath]] From eb8e4fad2986fa117e995eaa87a57ec52f724634 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 25 Oct 2017 15:29:58 +0100 Subject: [PATCH 02/16] Add docs on how to manually create state graph --- doc/using-drafter.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/using-drafter.md diff --git a/doc/using-drafter.md b/doc/using-drafter.md new file mode 100644 index 000000000..d7c3d1b73 --- /dev/null +++ b/doc/using-drafter.md @@ -0,0 +1,21 @@ +# Using Drafter + +## Convert any existing Grafts to work with Drafter + +If you have an existing Database with quads in it you may need to make drafter aware of them. To do this you can run the update statement though be careful not to run it twice on the same database! + +```sparql +INSERT { + GRAPH { + ?g a . + ?g true . + } +} WHERE { + SELECT DISTINCT ?g + WHERE { + GRAPH ?g { + ?s ?p ?o . + } + } +} +``` From 303c21666afe4a858c1234a35f695598b37684d3 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 25 Oct 2017 15:30:30 +0100 Subject: [PATCH 03/16] Update using-drafter.md --- doc/using-drafter.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/using-drafter.md b/doc/using-drafter.md index d7c3d1b73..5e73d2e61 100644 --- a/doc/using-drafter.md +++ b/doc/using-drafter.md @@ -1,5 +1,7 @@ # Using Drafter +`TODO` Make this more of a guide on how to run drafter as a user etc. + ## Convert any existing Grafts to work with Drafter If you have an existing Database with quads in it you may need to make drafter aware of them. To do this you can run the update statement though be careful not to run it twice on the same database! From b062fdb74bba5a12ac6ee87b4ad2020a9490e105 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 25 Oct 2017 15:33:02 +0100 Subject: [PATCH 04/16] Add link to using drafter docs --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 75a3e08d8..8c8760ca6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ A RESTful Clojure web service to support PMD's admin tool in moving data updates between draft and live triple stores. -Be sure to see the [Getting Started Guide](https://github.com/Swirrl/drafter/blob/master/doc/getting-started.org) for how to use Drafter and set up your Dev environment. +- Using Drafter as part of PMD or otherwise? Then see the [User Guide](https://github.com/Swirrl/drafter/blob/master/doc/using-drafter.md) +- Developing on Drafter itself? Then see the [Getting Started Guide](https://github.com/Swirrl/drafter/blob/master/doc/getting-started.org) for how to use Drafter and set up your Dev environment. ## Configuring Drafter From b4dcd5aab49d8c7a3102c1218a214fcf4b67e499 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Fri, 19 Jan 2018 14:51:11 +0000 Subject: [PATCH 05/16] Bump version to 2.1.9 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index bad3dc334..155847828 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject drafter "2.1.8-SNAPSHOT" +(defproject drafter "2.1.9" :description "Backend PMD service" :url "http://github.com/Swirrl/drafter" :license {:name "Proprietary & Commercially Licensed Only" From 2e22b73a1f76ad18911b1f3711f40a1a0addf461 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Fri, 26 Jan 2018 16:06:16 +0000 Subject: [PATCH 06/16] Investigating JENA-1468 empty values/rewriting bug https://issues.apache.org/jira/browse/JENA-1468?page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel&focusedCommentId=16332916#comment-16332916 --- .../test-queries/empty-values.sparql | 7 + project.clj | 8 +- src/drafter/rdf/rewriting/arq.clj | 165 ++++++++++++++---- src/drafter/rdf/rewriting/query_rewriting.clj | 9 + test/drafter/rdf/rewriting/arq_test.clj | 3 +- 5 files changed, 155 insertions(+), 37 deletions(-) create mode 100644 env/dev/resources/test-queries/empty-values.sparql diff --git a/env/dev/resources/test-queries/empty-values.sparql b/env/dev/resources/test-queries/empty-values.sparql new file mode 100644 index 000000000..0be937af3 --- /dev/null +++ b/env/dev/resources/test-queries/empty-values.sparql @@ -0,0 +1,7 @@ +SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +} diff --git a/project.clj b/project.clj index 155847828..ddef57d79 100644 --- a/project.clj +++ b/project.clj @@ -54,12 +54,12 @@ [metosin/ring-swagger-ui "2.2.10"] ;; Use JENA for our query rewriting - [org.apache.jena/jena-arq "3.4.0" :exclusions [org.slf4j/slf4j-api + [org.apache.jena/jena-arq "3.6.0" :exclusions [org.slf4j/slf4j-api org.slf4j/jcl-over-slf4j org.apache.httpcomponents/httpclient]] - [org.apache.jena/jena-base "3.4.0" :exclusions [org.slf4j/slf4j-api]] - [org.apache.jena/jena-core "3.4.0" :exclusions [org.slf4j/slf4j-api]] - [org.apache.jena/jena-iri "3.4.0" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-base "3.6.0" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-core "3.6.0" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-iri "3.6.0" :exclusions [org.slf4j/slf4j-api]] [org.mindrot/jbcrypt "0.4"] diff --git a/src/drafter/rdf/rewriting/arq.clj b/src/drafter/rdf/rewriting/arq.clj index 260c56e67..b20dc62fa 100644 --- a/src/drafter/rdf/rewriting/arq.clj +++ b/src/drafter/rdf/rewriting/arq.clj @@ -67,7 +67,7 @@ (.isList i))) (defn sse-zipper - "Construct a zipper on a Sparql SExpression Item, that allows easy persistent + "Construct a zipper on a Sparql SExpression Item, that allows easy walking of the sparql algebra tree." [sse-item] (z/zipper sse-list-item? @@ -104,15 +104,36 @@ in the SSE algebra tree, such as the set of prefixes and the query type." [rewriter qstr] - (let [q (sparql-string->arq-query qstr) - transform-query (fn [q] - (-> q - ->sse-item - sse-zipper - rewriter - str - (SSE/parseOp (.getPrefixMapping q)) - OpAsQuery/asQuery))] + (let [q (QueryFactory/create qstr Syntax/syntaxSPARQL_11) + ;q (sparql-string->arq-query qstr) + transform-query (fn [q] + (-> q + ->sse-item + sse-zipper + rewriter + str + (SSE/parseOp (.getPrefixMapping q)) + OpAsQuery/asQuery) + + #_(-> q + ->sse-item + sse-zipper + rewriter + + + org.apache.jena.sparql.sse.builders.BuilderOp/build + + + OpAsQuery/asQuery) + + #_(->> (-> q + + Algebra/compile + sse-zipper) + + rewriter + org.apache.jena.sparql.sse.builders.BuilderOp/build + OpAsQuery/asQuery))] (doto (cond ;; NOTE that to support primitive DESCRIBE queries such as those of @@ -155,6 +176,14 @@ (comment + (println (str (->sse-item (sparql-string->arq-query "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +}")))) + ;; Playing with zippers (let [item (->sse-item (sparql-string->arq-query "SELECT * WHERE { GRAPH { ?s ?p ?o } GRAPH { ?s ?p ?o } }"))] @@ -174,34 +203,106 @@ SSE/parseOp OpAsQuery/asQuery)) -;; constructs are different... -(str ( (query->rewritable "PREFIX : CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") identity)) + ;; constructs are different... + (str ( (query->rewritable "PREFIX : CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") identity)) -;; NOTE it returns a SELECT... weird... I think it's because select/construct/ask is decided outside the algebra -;; => "PREFIX : \n\nSELECT *\nWHERE\n { ?s ?p ?o .\n ?s2 ?p ?o2\n }\n" + ;; NOTE it returns a SELECT... weird... I think it's because select/construct/ask is decided outside the algebra + ;; => "PREFIX : \n\nSELECT *\nWHERE\n { ?s ?p ?o .\n ?s2 ?p ?o2\n }\n" -;; https://jena.apache.org/documentation/query/algebra.html + ;; https://jena.apache.org/documentation/query/algebra.html -;; round tripping from sparql string -> sse algebra -> sparql string -(-> (QueryFactory/create "PREFIX dcterms: SELECT * WHERE { ?s ?p ?o }") - Algebra/compile - OpAsQuery/asQuery) + ;; round tripping from sparql string -> sse algebra -> sparql string + (-> (QueryFactory/create "PREFIX dcterms: SELECT * WHERE { ?s ?p ?o }") + Algebra/compile + OpAsQuery/asQuery) -(-> (QueryFactory/create "PREFIX dcterms: SELECT * WHERE { ?s ?p ?o }") - Algebra/compile - OpAsQuery/asQuery - str - SSE/parseItem - .getList - seq) + (-> (QueryFactory/create "PREFIX dcterms: SELECT * WHERE { ?s ?p ?o }") + Algebra/compile + OpAsQuery/asQuery + str + SSE/parseItem + .getList + seq) -(QueryFactory/create "PREFIX : CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") + (QueryFactory/create "PREFIX : CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o . ?s2 ?p ?o2 }") -(Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")) + (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")) -(Algebra/optimize (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }"))) + (Algebra/optimize (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }"))) -(Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")) + + (Algebra/compile (QueryFactory/create "SELECT ?s WHERE { GRAPH <:foo> { ?s ?p ?o } GRAPH <:foo> { ?s ?p ?o } }")) -(last (.getList (SSE/parseItem "(graph :foo (bgp (triple ?s ?p ?o)))"))) -) + (last (.getList (SSE/parseItem "(graph :foo (bgp (triple ?s ?p ?o)))"))) + + + + ;; (OpAsQuery/asQuery (drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {} (Algebra/compile (sparql-string->arq-query "SELECT DISTINCT ?mdg + ;; WHERE { + ;; VALUES ?mdg { } + ;; GRAPH ?mdg { + ;; ?ds . + ;; } + ;; } + ;; ")))) + + ;; simplified + (->> (-> + "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +} +" + sparql-string->arq-query + Algebra/compile + sse-zipper + ) + (drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {} ) + OpAsQuery/asQuery) + + + (def mdg-query "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +} +") + + ;; expanded + (->> (-> + mdg-query + (QueryFactory/create Syntax/syntaxSPARQL_11) + Algebra/compile + sse-zipper + ) + + (drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {} ) + OpAsQuery/asQuery) + ;; above works wtf... + + (-> mdg-query + sparql-string->arq-query + ->sse-item + sse-zipper + z/root + ;#_str + ;#_(SSE/parseOp (.getPrefixMapping q)) + ;#_OpAsQuery/asQuery + + + org.apache.jena.sparql.sse.builders.BuilderOp/build + + OpAsQuery/asQuery) + + + (apply-rewriter (partial drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {}) mdg-query) + + + (OpAsQuery/asQuery (Algebra/compile (QueryFactory/create "SELECT ?foo WHERE { ?foo ?bar ?baz . VALUES ?foo {} }"))) + + ) diff --git a/src/drafter/rdf/rewriting/query_rewriting.clj b/src/drafter/rdf/rewriting/query_rewriting.clj index 1c07500bd..f4580c71e 100644 --- a/src/drafter/rdf/rewriting/query_rewriting.clj +++ b/src/drafter/rdf/rewriting/query_rewriting.clj @@ -57,5 +57,14 @@ (apply-rewriter (partial uri-constant-rewriter {"http://foo.com/" "http://bar.com/"}) "SELECT * WHERE { GRAPH { ?s ?p ?o }}") + (apply-rewriter (partial uri-constant-rewriter {}) + "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +}") + ) diff --git a/test/drafter/rdf/rewriting/arq_test.clj b/test/drafter/rdf/rewriting/arq_test.clj index ff4951d9e..ad8ddb886 100644 --- a/test/drafter/rdf/rewriting/arq_test.clj +++ b/test/drafter/rdf/rewriting/arq_test.clj @@ -33,4 +33,5 @@ "Round tripping twice generates same query string"))) (t/deftest rewriting-with-sparql-group-concats - (t/is (roundtripable? (load-query "test-queries/group-concat.sparql")))) + (t/is (roundtripable? (load-query "test-queries/group-concat.sparql"))) + (t/is (roundtripable? (load-query "test-queries/empty-values.sparql")))) From c95df58f3e96ae862a82186a364d87c47c99a13e Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Mon, 29 Jan 2018 15:06:57 +0000 Subject: [PATCH 07/16] Bump to SNAPSHOT/patched version of JENA 3.7.0-SNAPSHOT. To build this patch locally you will need to build a JENA snapshot by running: `$ mvn install` On this branch: https://github.com/afs/jena/tree/2586abf09751fd962ca7afe25d4ab3d9431ce716 Also add tests to prove issue is resolved. --- .../resources/test-queries/empty-values.sparql | 2 +- project.clj | 17 +++++++++++++---- src/drafter/rdf/rewriting/arq.clj | 15 ++++++++++++++- test/drafter/rdf/rewriting/arq_test.clj | 13 +++++++++++++ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/env/dev/resources/test-queries/empty-values.sparql b/env/dev/resources/test-queries/empty-values.sparql index 0be937af3..63118153c 100644 --- a/env/dev/resources/test-queries/empty-values.sparql +++ b/env/dev/resources/test-queries/empty-values.sparql @@ -2,6 +2,6 @@ SELECT DISTINCT ?mdg WHERE { VALUES ?mdg { } GRAPH ?mdg { - ?ds . + ?ds a } } diff --git a/project.clj b/project.clj index ddef57d79..7ff05611e 100644 --- a/project.clj +++ b/project.clj @@ -53,13 +53,22 @@ [metosin/ring-swagger-ui "2.2.10"] + ;; + ;; Bump JENA to 3.7.0-SNAPSHOT based on this commit: + ;; https://github.com/afs/jena/tree/2586abf09751fd962ca7afe25d4ab3d9431ce716 + ;; + ;; To test upstream fixes for https://issues.apache.org/jira/browse/JENA-1468 + ;; + ;; When this patch gets deployed to mainline jena we should be able to + ;; revert: https://github.com/Swirrl/publish_my_data_2/pull/763 + ;; ;; Use JENA for our query rewriting - [org.apache.jena/jena-arq "3.6.0" :exclusions [org.slf4j/slf4j-api + [org.apache.jena/jena-arq "3.7.0-SNAPSHOT" :exclusions [org.slf4j/slf4j-api org.slf4j/jcl-over-slf4j org.apache.httpcomponents/httpclient]] - [org.apache.jena/jena-base "3.6.0" :exclusions [org.slf4j/slf4j-api]] - [org.apache.jena/jena-core "3.6.0" :exclusions [org.slf4j/slf4j-api]] - [org.apache.jena/jena-iri "3.6.0" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-base "3.7.0-SNAPSHOT" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-core "3.7.0-SNAPSHOT" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-iri "3.7.0-SNAPSHOT" :exclusions [org.slf4j/slf4j-api]] [org.mindrot/jbcrypt "0.4"] diff --git a/src/drafter/rdf/rewriting/arq.clj b/src/drafter/rdf/rewriting/arq.clj index b20dc62fa..7145b7dcb 100644 --- a/src/drafter/rdf/rewriting/arq.clj +++ b/src/drafter/rdf/rewriting/arq.clj @@ -300,7 +300,20 @@ WHERE { OpAsQuery/asQuery) - (apply-rewriter (partial drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {}) mdg-query) + (-> + (partial drafter.rdf.rewriting.query-rewriting/uri-constant-rewriter {}) + (apply-rewriter "SELECT DISTINCT ?mdg +WHERE { + VALUES ?mdg { } + GRAPH ?mdg { + ?ds . + } +} +") + str) + + (println "SELECT DISTINCT ?mdg\nWHERE\n { VALUES ?mdg { }\n GRAPH ?mdg\n { ?ds a }\n }\n") + (OpAsQuery/asQuery (Algebra/compile (QueryFactory/create "SELECT ?foo WHERE { ?foo ?bar ?baz . VALUES ?foo {} }"))) diff --git a/test/drafter/rdf/rewriting/arq_test.clj b/test/drafter/rdf/rewriting/arq_test.clj index ad8ddb886..3010c6823 100644 --- a/test/drafter/rdf/rewriting/arq_test.clj +++ b/test/drafter/rdf/rewriting/arq_test.clj @@ -2,6 +2,7 @@ "Testing round tripping through Jena ARC" (:require [clojure.java.io :as io] [clojure.test :as t] + [clojure.string :as str] [drafter.rdf.rewriting.arq :as sut])) (defn load-query [res-path] @@ -35,3 +36,15 @@ (t/deftest rewriting-with-sparql-group-concats (t/is (roundtripable? (load-query "test-queries/group-concat.sparql"))) (t/is (roundtripable? (load-query "test-queries/empty-values.sparql")))) + +(defn normalise-whitespace + "Replace all consecutive whitespace with a single space." + [q] + (str/replace q #"\s+" " ")) + +(t/deftest preserves-empty-values-blocks + (let [normalised-query (normalise-whitespace (load-query "test-queries/empty-values.sparql"))] + (t/is (= normalised-query (normalise-whitespace (round-trip-query-string normalised-query))) + "Empty Values blocks are preserved"))) + + From 7e9e2b7aae2f93d32dcf022faefe8128a2064b84 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 31 Jan 2018 10:52:30 +0000 Subject: [PATCH 08/16] Lock jena to a dev build that fixes values clause rewriting JENA-1468 --- project.clj | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/project.clj b/project.clj index 7ff05611e..23870a6d6 100644 --- a/project.clj +++ b/project.clj @@ -15,6 +15,11 @@ ;; :passphrase :env ;; :snapshots false}]] + :repositories [["apache-dev" {:url "https://repository.apache.org/content/repositories/snapshots/" + :releases false}] + + ] + :classifiers {:prod :prod :dev :dev} @@ -53,23 +58,21 @@ [metosin/ring-swagger-ui "2.2.10"] - ;; - ;; Bump JENA to 3.7.0-SNAPSHOT based on this commit: + ;; Use JENA for our query rewriting + + + ;; Lock JENA to a specific 3.7.0-SNAPSHOT based on this commit: ;; https://github.com/afs/jena/tree/2586abf09751fd962ca7afe25d4ab3d9431ce716 ;; - ;; To test upstream fixes for https://issues.apache.org/jira/browse/JENA-1468 - ;; - ;; When this patch gets deployed to mainline jena we should be able to - ;; revert: https://github.com/Swirrl/publish_my_data_2/pull/763 - ;; - ;; Use JENA for our query rewriting - [org.apache.jena/jena-arq "3.7.0-SNAPSHOT" :exclusions [org.slf4j/slf4j-api - org.slf4j/jcl-over-slf4j - org.apache.httpcomponents/httpclient]] - [org.apache.jena/jena-base "3.7.0-SNAPSHOT" :exclusions [org.slf4j/slf4j-api]] - [org.apache.jena/jena-core "3.7.0-SNAPSHOT" :exclusions [org.slf4j/slf4j-api]] - [org.apache.jena/jena-iri "3.7.0-SNAPSHOT" :exclusions [org.slf4j/slf4j-api]] + ;; We should update these deps when 3.7.0 is released. + [org.apache.jena/jena-arq "3.7.0-20180131.100034-17" :exclusions [org.slf4j/slf4j-api + org.slf4j/jcl-over-slf4j + org.apache.httpcomponents/httpclient]] + [org.apache.jena/jena-base "3.7.0-20180131.095758-17" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-core "3.7.0-20180131.095918-17" :exclusions [org.slf4j/slf4j-api]] + [org.apache.jena/jena-iri "3.7.0-20180131.095726-17" :exclusions [org.slf4j/slf4j-api]] + [org.mindrot/jbcrypt "0.4"] [org.slf4j/slf4j-log4j12 "1.7.25" :exclusions [log4j org.slf4j/slf4j-api]] From 0346642d815e3c007c01583f4df90e4fba7ed516 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 31 Jan 2018 10:53:35 +0000 Subject: [PATCH 09/16] Fix broken test --- test/drafter/write_scheduler_test.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/drafter/write_scheduler_test.clj b/test/drafter/write_scheduler_test.clj index 7cffe50ff..8e112894e 100644 --- a/test/drafter/write_scheduler_test.clj +++ b/test/drafter/write_scheduler_test.clj @@ -60,7 +60,7 @@ (let [response (resp/submit-async-job! (const-job :background-write :done))] (is (= 202 (:status response)) "Job returns 202 (Accepted)") - (is (= (string? (get-in response [:body :finished-job]))) + (is (string? (get-in response [:body :finished-job])) "Job executes and returns its value"))) (testing "when global-writes-lock is locked" @@ -85,7 +85,7 @@ (let [response (resp/submit-async-job! (const-job :publish-write :done))] (is (= 202 (:status response)) "Job returns 202 (Accepted)") - (is (= (string? (get-in response [:body :finished-job]))) + (is (string? (get-in response [:body :finished-job])) "Job executes and returns its value"))) (testing "when global-writes-lock is locked" From 75508275ae5fe56488543c61e665e450bcd169cb Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 31 Jan 2018 10:58:50 +0000 Subject: [PATCH 10/16] Bump version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 23870a6d6..908434b9f 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject drafter "2.1.9" +(defproject drafter "2.1.10" :description "Backend PMD service" :url "http://github.com/Swirrl/drafter" :license {:name "Proprietary & Commercially Licensed Only" From 881e3f7d908086762000bdc689983102024baeb6 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 31 Jan 2018 11:22:15 +0000 Subject: [PATCH 11/16] Add github release on tag deployer to travis config --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 41b003f7f..98569f2a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,3 +40,10 @@ deploy: on: repo: Swirrl/drafter branch: master + - provider: releases + api_key: "$RELEASE_OAUTH_TOKEN" + file: "target/drafter.jar" + skip_cleanup: true + on: + tags: true + condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ From db120b4626b43783fe493c17d6899da937165d44 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 31 Jan 2018 11:57:38 +0000 Subject: [PATCH 12/16] Update version to 2.1.12 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 908434b9f..83075bd10 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject drafter "2.1.10" +(defproject drafter "2.1.12" :description "Backend PMD service" :url "http://github.com/Swirrl/drafter" :license {:name "Proprietary & Commercially Licensed Only" From c1179b685d6df58597212c1640083db50778d6a0 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 31 Jan 2018 12:07:31 +0000 Subject: [PATCH 13/16] Tweak travis releases to have TAG and BUILD number in jar name --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 98569f2a5..f308f9126 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,8 @@ before_deploy: - mkdir -p dist - cp target/drafter.jar dist/drafter-$TRAVIS_BRANCH-latest.jar - cp target/drafter.jar dist/drafter-private-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER.jar +- mkdir releases +- cp target/drafter.jar releases/drafter-$TRAVIS_TAG-$TRAVIS_BUILD_NUMBER.jar deploy: - provider: s3 access_key_id: AKIAJZVFELTIBDDYV5OQ @@ -42,7 +44,7 @@ deploy: branch: master - provider: releases api_key: "$RELEASE_OAUTH_TOKEN" - file: "target/drafter.jar" + file: "releases/drafter-$TRAVIS_TAG-$TRAVIS_BUILD_NUMBER.jar" skip_cleanup: true on: tags: true From 64b96f684811a0a7b05f3a6a0588f8b7faa8a72a Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 31 Jan 2018 12:18:13 +0000 Subject: [PATCH 14/16] Force project.clj version to sync with git tag/travis version --- project.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 83075bd10..2a26a0efe 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,6 @@ -(defproject drafter "2.1.12" +(def VERSION (or (System/getenv "TRAVIS_TAG") "local-2.1.x-SNAPSHOT")) + +(defproject drafter VERSION :description "Backend PMD service" :url "http://github.com/Swirrl/drafter" :license {:name "Proprietary & Commercially Licensed Only" From 9c20fc55260c447a137d96b9155444d96bffbcd1 Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 31 Jan 2018 12:53:26 +0000 Subject: [PATCH 15/16] Allow travis/github releases from non release-number tags. Any tag of the form ^[a-z,A-Z]+.* (i.e. not a release number) should also be built, but have the branch-name and build number in the file name. --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index f308f9126..1f8110ad1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,3 +49,10 @@ deploy: on: tags: true condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ + - provider: releases + api_key: "$RELEASE_OAUTH_TOKEN" + file: "releases/drafter-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER.jar" + skip_cleanup: true + on: + tags: true + condition: $TRAVIS_TAG =~ ^[a-z,A-Z]+.* From ed1e65b9b2cdff4084e98e58d2fc95091996de9d Mon Sep 17 00:00:00 2001 From: Rick Moynihan Date: Wed, 31 Jan 2018 13:03:06 +0000 Subject: [PATCH 16/16] set "named" release/tags to be prereleases --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1f8110ad1..9de1f2238 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,7 @@ deploy: - provider: releases api_key: "$RELEASE_OAUTH_TOKEN" file: "releases/drafter-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER.jar" + prerelease: true skip_cleanup: true on: tags: true