feat(Nuvlabox): Bulk update support (#935)

* fix(Job): Set created-by attribute as user at the origin of the creation of the job

* fix(Job): Allow user to cancel created bulk jobs

* fix(Job): Move job utils and create new utils namespace and refactor to reuse code

* fix(Common utils): Enhance cannot do action because of state error message

code/src/com/sixsq/nuvla/server/resources/common/std_crud.clj

@@ -1,7 +1,6 @@
 (ns com.sixsq.nuvla.server.resources.common.std-crud
   "Standard CRUD functions for resources."
   (:require
-    [clojure.data.json :as json]
     [clojure.stacktrace :as st]
     [clojure.string :as str]
     [clojure.tools.logging :as log]
@@ -13,6 +12,7 @@
     [com.sixsq.nuvla.server.resources.common.crud :as crud]
     [com.sixsq.nuvla.server.resources.common.state-machine :as sm]
     [com.sixsq.nuvla.server.resources.common.utils :as u]
+    [com.sixsq.nuvla.server.resources.job.utils :as job-utils]
     [com.sixsq.nuvla.server.resources.spec.acl-collection :as acl-collection]
     [com.sixsq.nuvla.server.util.response :as r]))
 
@@ -177,41 +177,22 @@
           result  (db/bulk-delete resource-name options)]
       (r/json-response result))))
 
-(defn create-bulk-job
-  [action-name target-resource authn-info acl payload]
-  (let [json-payload   (-> payload
-                           (assoc :authn-info authn-info)
-                           (json/write-str))
-        create-request {:params      {:resource-name "job"}
-                        :body        {:action          action-name
-                                      :target-resource {:href target-resource}
-                                      :payload         json-payload
-                                      :acl             acl}
-                        :nuvla/authn auth/internal-identity}
-        {{job-id     :resource-id
-          job-status :status} :body} (crud/add create-request)]
-    (when (not= job-status 201)
-      (throw (r/ex-response
-               (str "unable to create async job for " action-name)
-               500 target-resource)))
-    (r/map-response (str "starting " action-name " with async " job-id)
-                    202 target-resource job-id)))
-
 (defn bulk-action-fn
   [resource-name collection-acl _collection-uri]
   (validate-collection-acl collection-acl)
   (fn [{:keys [params body] :as request}]
     (throw-bulk-header-missing request)
     (throw-bulk-require-cimi-filter request)
     (a/throw-cannot-bulk-action collection-acl request)
-    (let [authn-info  (auth/current-authentication request)
-          acl         {:owners   ["group/nuvla-admin"]
-                       :view-acl [(auth/current-active-claim request)]}
-          action-name (-> params
-                          :action
-                          (str/replace #"-" "_")
-                          (str "_" resource-name))]
-      (create-bulk-job action-name resource-name authn-info acl body))))
+    (let [active-claim (auth/current-active-claim request)
+          acl          {:owners   ["group/nuvla-admin"]
+                        :view-acl [active-claim]
+                        :manage   [active-claim]}
+          action-name  (-> params
+                           :action
+                           (str/replace #"-" "_")
+                           (str "_" resource-name))]
+      (job-utils/create-bulk-job action-name resource-name request acl body))))
 
 (defn add-metric-fn
   [resource-name collection-acl _resource-uri & {:keys [validate-fn options]}]
@@ -228,8 +209,8 @@
     (throw-bulk-header-missing request)
     (a/throw-cannot-add collection-acl request)
     (a/throw-cannot-bulk-action collection-acl request)
-    (let [options    (select-keys request [:nuvla/authn :body])
-          response   (db/bulk-insert-metrics resource-name body options)]
+    (let [options  (select-keys request [:nuvla/authn :body])
+          response (db/bulk-insert-metrics resource-name body options)]
       (r/json-response response))))
 
 (defn generic-bulk-operation-fn

code/src/com/sixsq/nuvla/server/resources/common/utils.clj

@@ -371,22 +371,19 @@
   [state resource]
   (= (:state resource) state))
 
-(def is-not-in-state? (complement is-state?))
-
-
-(defn throw-can-not-do
+(defn throw-cannot-do
   [{:keys [id] :as resource} pred error-msg]
   (if (pred resource)
     resource
     (throw (r/ex-response error-msg 409 id))))
 
-(defn throw-can-not-do-action
+(defn throw-cannot-do-action
   [{:keys [id] :as resource} pred action]
-  (throw-can-not-do resource pred (format "operation '%s' not allowed on %s" action id)))
+  (throw-cannot-do resource pred (format "operation '%s' not allowed on %s" action id)))
 
-(defn throw-can-not-do-action-invalid-state
+(defn throw-cannot-do-action-invalid-state
   [{:keys [id state] :as resource} pred action]
-  (throw-can-not-do resource pred (format "invalid state (%s) for %s on %s" state action id)))
+  (throw-cannot-do resource pred (format "Action %s cannot be called on %s in state %s!" action id state)))
 
 (defn filter-eq-vals
   [attribute vals]

code/src/com/sixsq/nuvla/server/resources/credential.clj

@@ -10,7 +10,7 @@ passwords) or other services (e.g. TLS credentials for Docker). Creating new
     [com.sixsq.nuvla.server.resources.common.crud :as crud]
     [com.sixsq.nuvla.server.resources.common.std-crud :as std-crud]
     [com.sixsq.nuvla.server.resources.common.utils :as u]
-    [com.sixsq.nuvla.server.resources.job :as job]
+    [com.sixsq.nuvla.server.resources.job.utils :as job-utils]
     [com.sixsq.nuvla.server.resources.resource-metadata :as md]
     [com.sixsq.nuvla.server.resources.spec.credential :as credential]
     [com.sixsq.nuvla.server.util.log :as logu]
@@ -138,10 +138,11 @@ passwords) or other services (e.g. TLS credentials for Docker). Creating new
       (if-let [active-claim (auth/current-active-claim request)]
         (let [job-type "credential_check"
               {{job-id     :resource-id
-                job-status :status} :body} (job/create-job id job-type
-                                                           {:owners   ["group/nuvla-admin"]
-                                                            :view-acl [active-claim]}
-                                                           :priority 50)
+                job-status :status} :body} (job-utils/create-job id job-type
+                                                                 {:owners   ["group/nuvla-admin"]
+                                                                  :view-acl [active-claim]}
+                                                                 (auth/current-user-id request)
+                                                                 :priority 50)
               job-msg  (str "starting " id " with async " job-id)]
           (when (not= job-status 201)
             (throw (r/ex-response (format "unable to create async job to %s" job-type) 500 id)))

code/src/com/sixsq/nuvla/server/resources/deployment.clj

@@ -213,7 +213,7 @@ a container orchestration engine.
            deployment    (-> (crud/retrieve-by-id-as-admin deployment-id)
                              (a/throw-cannot-delete request)
                              (cond-> (not force-delete)
-                                     (u/throw-can-not-do-action-invalid-state
+                                     (u/throw-cannot-do-action-invalid-state
                                        utils/can-delete? "delete")))]
        (ectx/add-to-context :acl (:acl deployment))
        (ectx/add-to-context :resource deployment)
@@ -286,7 +286,7 @@ a container orchestration engine.
   (try
     (let [id             (str resource-type "/" uuid)
           deployment     (-> (crud/retrieve-by-id-as-admin id)
-                             (u/throw-can-not-do-action-invalid-state utils/can-start? "start")
+                             (u/throw-cannot-do-action-invalid-state utils/can-start? "start")
                              (utils/throw-when-payment-required request)
                              (utils/throw-can-not-access-registries-creds request)
                              (utils/throw-can-not-access-helm-repo-cred request)
@@ -316,7 +316,7 @@ a container orchestration engine.
   (try
     (let [deployment     (-> (str resource-type "/" uuid)
                              (crud/retrieve-by-id-as-admin)
-                             (u/throw-can-not-do-action-invalid-state utils/can-stop? "stop"))
+                             (u/throw-cannot-do-action-invalid-state utils/can-stop? "stop"))
           execution-mode (:execution-mode deployment)]
       (-> deployment
           (assoc :state "STOPPING")
@@ -334,7 +334,7 @@ a container orchestration engine.
     (-> (str resource-type "/" uuid)
         (crud/retrieve-by-id-as-admin)
         (a/throw-cannot-manage request)
-        (u/throw-can-not-do-action-invalid-state utils/can-create-log? "create-log")
+        (u/throw-cannot-do-action-invalid-state utils/can-create-log? "create-log")
         (utils/throw-when-payment-required request)
         (utils/create-log request))
     (catch Exception e
@@ -372,7 +372,7 @@ a container orchestration engine.
     (let [current (-> (str resource-type "/" uuid)
                       (crud/retrieve-by-id-as-admin)
                       (a/throw-cannot-manage request)
-                      (u/throw-can-not-do-action-invalid-state
+                      (u/throw-cannot-do-action-invalid-state
                         utils/can-update? "update_deployment")
                       (utils/throw-when-payment-required request)
                       (utils/throw-can-not-access-registries-creds request)
@@ -394,7 +394,7 @@ a container orchestration engine.
   (try
     (-> (str resource-type "/" uuid)
         (crud/retrieve-by-id-as-admin)
-        (u/throw-can-not-do-action utils/can-detach? "detach")
+        (u/throw-cannot-do-action utils/can-detach? "detach")
         (dissoc :deployment-set :deployment-set-name)
         u/update-timestamps
         db/edit)

code/src/com/sixsq/nuvla/server/resources/deployment/utils.clj

@@ -14,8 +14,8 @@
     [com.sixsq.nuvla.server.resources.configuration-nuvla :as config-nuvla]
     [com.sixsq.nuvla.server.resources.credential :as credential]
     [com.sixsq.nuvla.server.resources.credential-template-api-key :as cred-api-key]
-    [com.sixsq.nuvla.server.resources.job :as job]
     [com.sixsq.nuvla.server.resources.job.interface :as job-interface]
+    [com.sixsq.nuvla.server.resources.job.utils :as job-utils]
     [com.sixsq.nuvla.server.resources.module.utils :as module-utils]
     [com.sixsq.nuvla.server.resources.nuvlabox.utils :as nuvlabox-utils]
     [com.sixsq.nuvla.server.resources.resource-log :as resource-log]
@@ -113,13 +113,14 @@
         low-priority (get-in request [:body :low-priority] false)
         parent-job   (get-in request [:body :parent-job])
         {{job-id     :resource-id
-          job-status :status} :body} (job/create-job
+          job-status :status} :body} (job-utils/create-job
                                        id action
                                        (-> {:owners ["group/nuvla-admin"]}
                                            (a/acl-append :edit-data nuvlabox)
                                            (a/acl-append :manage nuvlabox)
                                            (a/acl-append :view-data active-claim)
                                            (a/acl-append :manage active-claim))
+                                       (auth/current-user-id request)
                                        :parent-job parent-job
                                        :priority (if low-priority 999 50)
                                        :execution-mode execution-mode
@@ -225,11 +226,11 @@
                                     (map crud/retrieve-by-id-as-admin)))
         registries-infra (when full
                            (map (comp crud/retrieve-by-id-as-admin :parent) registries-creds))
-        module-content (some-> deployment :module :content)
+        module-content   (some-> deployment :module :content)
         helm-repo-cred   (some-> module-content :helm-repo-cred
                                  crud/retrieve-by-id-as-admin)
-        helm-repo-url (some-> module-content :helm-repo-url
-                              crud/retrieve-by-id-as-admin)]
+        helm-repo-url    (some-> module-content :helm-repo-url
+                                 crud/retrieve-by-id-as-admin)]
     (job-interface/get-context->response
       deployment
       credential

code/src/com/sixsq/nuvla/server/resources/deployment_set.clj

@@ -6,15 +6,13 @@ These resources represent a deployment set that regroups deployments.
     [clojure.tools.logging :as log]
     [com.sixsq.nuvla.auth.acl-resource :as a]
     [com.sixsq.nuvla.auth.utils :as auth]
-    [com.sixsq.nuvla.db.filter.parser :as parser]
     [com.sixsq.nuvla.db.impl :as db]
     [com.sixsq.nuvla.server.resources.common.crud :as crud]
     [com.sixsq.nuvla.server.resources.common.state-machine :as sm]
     [com.sixsq.nuvla.server.resources.common.std-crud :as std-crud]
     [com.sixsq.nuvla.server.resources.common.utils :as u]
     [com.sixsq.nuvla.server.resources.deployment-set.operational-status :as os]
     [com.sixsq.nuvla.server.resources.deployment-set.utils :as utils]
-    [com.sixsq.nuvla.server.resources.job :as job]
     [com.sixsq.nuvla.server.resources.job.interface :as job-interface]
     [com.sixsq.nuvla.server.resources.job.utils :as job-utils]
     [com.sixsq.nuvla.server.resources.module :as module]
@@ -192,19 +190,19 @@ These resources represent a deployment set that regroups deployments.
 
 (defn action-bulk
   [{:keys [id] :as _resource} {{:keys [action]} :params :as request}]
-  (let [authn-info (auth/current-authentication request)
-        acl        {:owners   ["group/nuvla-admin"]
-                    :view-acl [(auth/current-active-claim request)]}]
-    (std-crud/create-bulk-job
-      (utils/bulk-action-job-name action) id authn-info acl {})))
+  (let [acl {:owners   ["group/nuvla-admin"]
+             :view-acl [(auth/current-active-claim request)]}]
+    (job-utils/create-bulk-job
+      (utils/bulk-action-job-name action) id request acl {})))
 
 (defn action-simple
   [{:keys [id] :as _resource} {{:keys [action]} :params :as request}]
   (let [job-action (utils/action-job-name action)
         {{job-id     :resource-id
-          job-status :status} :body} (job/create-job id (utils/action-job-name action)
-                                                     {:owners   ["group/nuvla-admin"]
-                                                      :view-acl [(auth/current-active-claim request)]})
+          job-status :status} :body} (job-utils/create-job id (utils/action-job-name action)
+                                                           {:owners   ["group/nuvla-admin"]
+                                                            :view-acl [(auth/current-active-claim request)]}
+                                                           (auth/current-user-id request))
         job-msg    (str action " on " id " with async " job-id)]
     (if (not= job-status 201)
       (throw (r/ex-response (format "unable to create async job to %s" job-action) 500 id))
@@ -352,18 +350,10 @@ These resources represent a deployment set that regroups deployments.
 
 (defn cancel-latest-job
   [{:keys [id] :as _resource} _request]
-  (let [filter-str (format "target-resource/href='%s' and (state='%s' or state='%s')" id
-                           job-utils/state-queued job-utils/state-running)
-        [_ [{job-id :id}]]
-        (crud/query-as-admin
-          job/resource-type
-          {:cimi-params {:filter  (parser/parse-cimi-filter filter-str)
-                         :orderby [["created" :desc]]
-                         :last    1}})]
-    (if job-id
-      (do (crud/do-action-as-admin job-id job-utils/action-cancel)
-          (r/map-response "operation cancelled" 200))
-      (r/map-response "no running operation found that can be cancelled" 404))))
+  (if-let [job-id (job-utils/existing-job-id-not-in-final-state id)]
+    (do (crud/do-action-as-admin job-id job-utils/action-cancel)
+        (r/map-response "operation cancelled" 200))
+    (r/map-response "no running operation found that can be cancelled" 404)))
 
 (defmethod crud/do-action [resource-type utils/action-cancel]
   [request]

code/src/com/sixsq/nuvla/server/resources/job.clj

@@ -25,7 +25,7 @@ request.
     [com.sixsq.nuvla.server.util.metadata :as gen-md]))
 
 
-(def ^:const resource-type (u/ns->type *ns*))
+(def ^:const resource-type utils/resource-type)
 
 
 (def ^:const collection-type (u/ns->collection-type *ns*))
@@ -77,9 +77,10 @@ request.
 ;; CRUD operations
 ;;
 
-(defn add-impl [{{:keys [priority execution-mode version]
-                  :or {priority 999 execution-mode "push"
-                       version  latest-version} :as body} :body :as request}]
+(defn add-impl [{{:keys [priority execution-mode version created-by]
+                  :or {priority 999
+                       execution-mode "push"
+                       version latest-version} :as body} :body :as request}]
   (a/throw-cannot-add collection-acl request)
   (let [id      (u/new-resource-id resource-type)
         zk-path (when (#{"push" "mixed"} execution-mode)
@@ -94,7 +95,7 @@ request.
         (assoc :execution-mode execution-mode)
         (assoc :version version)
         u/update-timestamps
-        (u/set-created-by request)
+        (cond-> (nil? created-by) (u/set-created-by request))
         utils/job-cond->addition
         (crud/add-acl request)
         (cond-> zk-path (assoc :tags [zk-path]))
@@ -173,13 +174,15 @@ request.
             (utils/can-timeout? resource request) (update :operations conj timeout-op)
             (utils/can-get-context? resource request) (update :operations conj get-context-op))))
 
+(defn create-cancel-children-jobs-job
+  [{:keys [acl] parent-job-id :id :as _parent-job} request]
+  (utils/create-job parent-job-id "cancel_children_jobs" acl
+                       (auth/current-user-id request)
+                       :priority 10))
 
-(declare create-cancel-children-jobs-job)
-
-
-(defn cancel-children-jobs-async [{action :action :as job}]
+(defn cancel-children-jobs-async [{action :action :as job} request]
   (when (str/starts-with? action "bulk")
-    (create-cancel-children-jobs-job job)))
+    (create-cancel-children-jobs-job job request)))
 
 
 (defmethod crud/do-action [resource-type utils/action-cancel]
@@ -195,7 +198,7 @@ request.
                        (crud/validate)
                        (db/edit {:refresh false}))
           job      (:body response)]
-      (cancel-children-jobs-async job)
+      (cancel-children-jobs-async job (auth/current-active-claim request))
       (log/warn "Canceled job : " id)
       (interface/on-cancel job)
       response)
@@ -229,31 +232,3 @@ request.
       response)
     (catch Exception e
       (or (ex-data e) (throw e)))))
-
-;;
-;; internal crud
-;;
-
-(defn create-job
-  [target-resource action acl & {:keys [priority affected-resources
-                                        execution-mode payload
-                                        parent-job]}]
-  (let [job-map        (cond-> {:action          action
-                                :target-resource {:href target-resource}
-                                :acl             acl}
-                               priority (assoc :priority priority)
-                               parent-job (assoc :parent-job parent-job)
-                               affected-resources (assoc :affected-resources affected-resources)
-                               execution-mode (assoc :execution-mode execution-mode)
-                               payload (assoc :payload payload))
-        create-request {:params      {:resource-name resource-type}
-                        :body        job-map
-                        :nuvla/authn auth/internal-identity}]
-    (crud/add create-request)))
-
-
-(defn create-cancel-children-jobs-job
-  [{:keys [acl] parent-job-id :id :as _parent-job}]
-  (create-job parent-job-id "cancel_children_jobs" acl
-              :priority 10))
-