Merge branch 'develop'

commons-datastore/commons-datastore-core/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>3.5.1</version>
+        <version>3.6.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

commons-datastore/commons-datastore-core/src/main/java/org/opencb/commons/datastore/core/QueryResponse.java

@@ -36,6 +36,10 @@ public QueryResponse() {
         this("", -1, "", "", null, null);
     }
 
+    public QueryResponse(String apiVersion, int time, QueryOptions queryOptions, List<QueryResult<T>> response) {
+        this(apiVersion, time, "", "", queryOptions, response);
+    }
+
     public QueryResponse(QueryOptions queryOptions, List<QueryResult<T>> response) {
         this("", -1, "", "", queryOptions, response);
     }
@@ -125,47 +129,57 @@ public String toString() {
         return sb.toString();
     }
 
-    public void setApiVersion(String apiVersion) {
+    public String getApiVersion() {
+        return apiVersion;
+    }
+
+    public QueryResponse setApiVersion(String apiVersion) {
         this.apiVersion = apiVersion;
+        return this;
     }
 
-    public String getWarning() {
-        return warning;
+    public int getTime() {
+        return time;
     }
 
-    public void setTime(int time) {
+    public QueryResponse setTime(int time) {
         this.time = time;
+        return this;
     }
 
-    public String getApiVersion() {
-        return apiVersion;
+    public String getWarning() {
+        return warning;
     }
 
-    public void setWarning(String warning) {
+    public QueryResponse setWarning(String warning) {
         this.warning = warning;
+        return this;
     }
 
     public String getError() {
         return error;
     }
 
-    public void setError(String error) {
+    public QueryResponse setError(String error) {
         this.error = error;
+        return this;
     }
 
     public QueryOptions getQueryOptions() {
         return queryOptions;
     }
 
-    public void setQueryOptions(QueryOptions queryOptions) {
+    public QueryResponse setQueryOptions(QueryOptions queryOptions) {
         this.queryOptions = queryOptions;
+        return this;
     }
 
     public List<QueryResult<T>> getResponse() {
         return response;
     }
 
-    public void setResponse(List<QueryResult<T>> response) {
+    public QueryResponse setResponse(List<QueryResult<T>> response) {
         this.response = response;
+        return this;
     }
 }

commons-datastore/commons-datastore-mongodb/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>3.5.1</version>
+        <version>3.6.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

commons-datastore/commons-datastore-mongodb/src/main/java/org/opencb/commons/datastore/mongodb/GenericDocumentComplexConverter.java

@@ -78,6 +78,14 @@ public Document convertToStorageType(T object) {
         }
     }
 
+    public static Long getLongValue(Document document, String key) {
+        try {
+            return document.getLong(key);
+        } catch (ClassCastException e) {
+            return document.getInteger(key).longValue();
+        }
+    }
+
     /**
      * Replace all the dots in the keys with {@link #TO_REPLACE_DOTS}.
      *

commons-datastore/commons-datastore-mongodb/src/main/java/org/opencb/commons/datastore/mongodb/MongoDBCollection.java

@@ -27,7 +27,6 @@
 import com.mongodb.client.FindIterable;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoCursor;
-import com.mongodb.client.model.Aggregates;
 import com.mongodb.client.model.IndexOptions;
 import com.mongodb.client.result.DeleteResult;
 import com.mongodb.client.result.UpdateResult;
@@ -319,18 +318,8 @@ public <T> QueryResult<T> aggregate(List<? extends Bson> operations, ComplexType
 
         long start = startQuery();
 
-        // we need to be sure that the List is mutable
-        List<Bson> bsonOperations = new ArrayList<>(operations);
-        if (options != null) {
-            if (options.getInt(QueryOptions.SKIP) > 0) {
-                bsonOperations.add(Aggregates.skip(options.getInt(QueryOptions.SKIP)));
-            }
-            if (options.getInt(QueryOptions.LIMIT) > 0) {
-                bsonOperations.add(Aggregates.limit(options.getInt(QueryOptions.LIMIT)));
-            }
-        }
         QueryResult<T> queryResult;
-        AggregateIterable<Document> output = mongoDBNativeQuery.aggregate(bsonOperations, options);
+        AggregateIterable<Document> output = mongoDBNativeQuery.aggregate(operations, options);
         MongoCursor<Document> iterator = output.iterator();
         List<T> list = new LinkedList<>();
         if (queryResultWriter != null) {

commons-datastore/commons-datastore-mongodb/src/main/java/org/opencb/commons/datastore/mongodb/MongoDBNativeQuery.java

@@ -121,7 +121,17 @@ public FindIterable<Document> find(Bson query, Bson projection, QueryOptions opt
     }
 
     public AggregateIterable<Document> aggregate(List<? extends Bson> operations, QueryOptions options) {
-        return (operations.size() > 0) ? dbCollection.aggregate(operations) : null;
+        // we need to be sure that the List is mutable
+        List<Bson> bsonOperations = new ArrayList<>(operations);
+        if (options != null) {
+            if (options.getInt(QueryOptions.SKIP) > 0) {
+                bsonOperations.add(Aggregates.skip(options.getInt(QueryOptions.SKIP)));
+            }
+            if (options.getInt(QueryOptions.LIMIT) > 0) {
+                bsonOperations.add(Aggregates.limit(options.getInt(QueryOptions.LIMIT)));
+            }
+        }
+        return (bsonOperations.size() > 0) ? dbCollection.aggregate(bsonOperations) : null;
     }
 
     /**

commons-datastore/commons-datastore-mongodb/src/main/java/org/opencb/commons/datastore/mongodb/MongoDataStore.java

@@ -40,6 +40,7 @@
 
 public class MongoDataStore {
 
+    private static final String REPL_SET_KEY = "repl";
     private MongoClient mongoClient;
     private MongoDatabase db;
     private MongoDBConfiguration mongoDBConfiguration;
@@ -79,6 +80,15 @@ public Document getServerStatus() {
         return db.runCommand(new Document("serverStatus", 1));
     }
 
+    public Document getReplSetStatus() {
+        return db.runCommand(new Document("replSetGetStatus", 1));
+    }
+
+    public boolean isReplSet() {
+        Document document = getServerStatus();
+        return document.containsKey(REPL_SET_KEY);
+    }
+
     public MongoDBCollection createCollection(String collectionName) {
         if (!Arrays.asList(db.listCollectionNames()).contains(collectionName)) {
             db.createCollection(collectionName);

commons-datastore/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons</artifactId>
-        <version>3.5.1</version>
+        <version>3.6.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

commons-lib/pom.xml

@@ -7,7 +7,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons</artifactId>
-        <version>3.5.1</version>
+        <version>3.6.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -18,8 +18,8 @@
         <dependency>
             <groupId>com.beust</groupId>
             <artifactId>jcommander</artifactId>
-            <optional>true</optional>
-            <version>1.47</version>
+            <!--<optional>true</optional>-->
+            <version>1.64</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

commons-lib/src/main/java/org/opencb/commons/utils/CollectionUtils.java

@@ -0,0 +1,20 @@
+package org.opencb.commons.utils;
+
+import java.util.Collection;
+
+/**
+ * Created on 10/08/17.
+ *
+ * @author Jacobo Coll <jacobo167@gmail.com>
+ */
+public class CollectionUtils {
+
+    public static boolean isEmpty(Collection<?> collection) {
+        return collection == null || collection.isEmpty();
+    }
+
+    public static boolean isNotEmpty(Collection<?> collection) {
+        return !isEmpty(collection);
+    }
+
+}

commons-lib/src/main/java/org/opencb/commons/utils/CommandLineUtils.java

@@ -19,7 +19,10 @@
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.ParameterDescription;
 
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.PrintStream;
+import java.io.PrintWriter;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
 import java.util.*;
@@ -86,6 +89,116 @@ public static void printCommandUsage(JCommander commander, PrintStream printStre
         });
     }
 
+    /**
+     * Create the following strings from the jCommander to create AutoComplete bash script:
+     * commands="users projects studies files jobs individuals families samples variables cohorts alignments variant"
+     * users="create info update change-password delete projects login logout reset-password"
+     * ...
+     * users_create_options="--password --help --log-file --email --name --log-level --conf --organization --output-format --session-id"
+     * ...
+     * This accepts two levels: commands and subcommands
+     *
+     * @param jCommander    JComander object to extractt commands and subcommonds
+     * @param fileName      Filename destination
+     * @param bashFunctName Name of the bash function to autocomplete
+     * @param excludeParams List of params to be excluded
+     * @throws IOException If the destination file cannot be written
+     */
+    public static void generateBashAutoComplete(JCommander jCommander, String fileName, String bashFunctName, List<String> excludeParams)
+            throws IOException {
+        Map<String, JCommander> jCommands = jCommander.getCommands();
+        StringBuilder mainCommands = new StringBuilder();
+        StringBuilder subCommmands = new StringBuilder();
+        StringBuilder subCommmandsOptions = new StringBuilder();
+
+        // Create a HashSet to skip excluded parameters
+        Set<String> excludeParamsSet;
+        if (excludeParams == null) {
+            excludeParamsSet = new HashSet<>();
+        } else {
+            excludeParamsSet = new HashSet<>(excludeParams);
+        }
+
+        for (String command : jCommands.keySet()) {
+            JCommander subCommand = jCommands.get(command);
+            mainCommands.append(command).append(" ");
+            subCommmands.append(command + "=" + "\"");
+            Map<String, JCommander> subSubCommands = subCommand.getCommands();
+            for (String sc : subSubCommands.keySet()) {
+                subCommmands.append(sc).append(" ");
+                subCommmandsOptions.append(command + "_");
+                // - is not allowed in bash main variable name, replacing it with _
+                subCommmandsOptions.append(sc.replaceAll("[-.]+", "_") + "_" + "options=" + "\"");
+                JCommander subCommandOptions = subSubCommands.get(sc);
+                for (ParameterDescription param : subCommandOptions.getParameters()) {
+                    // Add parameter if it is not excluded
+                    if (!excludeParamsSet.contains(param.getLongestName())) {
+                        subCommmandsOptions.append(param.getLongestName()).append(' ');
+                    }
+                }
+                subCommmandsOptions.replace(0, subCommmandsOptions.length(), subCommmandsOptions.toString().trim()).append("\"" + "\n");
+            }
+            subCommmands.replace(0, subCommmands.length(), subCommmands.toString().trim()).append("\"" + "\n");
+        }
+
+        // Commands(Commands, subCommands and subCommandOptions) are populated intro three strings until this point,
+        // Now we write bash script commands and blend these strings into those as appropriate
+        StringBuilder autoComplete = new StringBuilder();
+        autoComplete.append("_" + bashFunctName + "() \n { \n local cur prev opts \n COMPREPLY=() \n cur="
+                + "$" + "{COMP_WORDS[COMP_CWORD]} \n prev=" + "$" + "{COMP_WORDS[COMP_CWORD-1]} \n");
+
+        autoComplete.append("commands=\"").append(mainCommands.toString().trim()).append('"').append('\n');
+        autoComplete.append(subCommmands.toString());
+        autoComplete.append(subCommmandsOptions.toString());
+        autoComplete.append("if [[ ${#COMP_WORDS[@]} > 2 && ${#COMP_WORDS[@]} < 4 ]] ; then \n local options \n case "
+                + "$" + "{prev} in \n");
+
+        for (String command : mainCommands.toString().split(" ")) {
+            autoComplete.append("\t" + command + ") options=" + "\"" + "${" + command + "}" + "\"" + " ;; \n");
+        }
+
+        autoComplete.append("*) ;; \n esac \n COMPREPLY=( $( compgen -W " + "\"" + "$" + "options" + "\""
+                + " -- ${cur}) ) \n return 0 \n elif [[ ${#COMP_WORDS[@]} > 3 ]] ; then \n local options \n case " + "$"
+                + "{COMP_WORDS[1]} in \n");
+
+        int subCommandIndex = 0;
+        for (String command : mainCommands.toString().split(" ")) {
+            String[] splittedSubCommands = subCommmands.toString().split("\n")[subCommandIndex]
+                    .replace(command + "=" + "\"", "").replace("\"", "").split(" ");
+
+            if (splittedSubCommands[0].isEmpty()) {
+                ++subCommandIndex;
+            } else {
+                autoComplete.append('\t').append(command).append(") \n");
+                autoComplete.append("\t\t case " + "$" + "{COMP_WORDS[2]} in \n");
+
+                for (String subCommand : splittedSubCommands) {
+                    autoComplete.append("\t\t").append(subCommand).append(") options=").append("\"").append("${").
+                            append(command).append("_").append(subCommand.replaceAll("[-.]+", "_")).append("_options}").
+                            append("\"").append(" ;; \n");
+                }
+                autoComplete.append("\t\t *) ;; esac ;; \n");
+                ++subCommandIndex;
+            }
+        }
+        autoComplete.append("*) ;;  esac \n COMPREPLY=( $( compgen -W " + "\"" + "$" + "options" + "\"" + " -- ${cur}) )"
+                + " \n return 0 \n fi \n if [[ ${cur} == * ]] ; then \n COMPREPLY=( $(compgen -W " + "\"" + "$"
+                + "{commands}" + "\"" + " -- ${cur}) ) \n return 0 \n fi \n } \n");
+
+        autoComplete.append("\n");
+        autoComplete.append("complete -F _" + bashFunctName + " " + bashFunctName + ".sh").append('\n');
+//        autoComplete.append("complete -F _" + bashFunctName + " ./bin/" + bashFunctName + ".sh").append('\n');
+//        autoComplete.append("complete -F _" + bashFunctName + " /opt/" + bashFunctName + "/bin/" + bashFunctName + ".sh").append('\n');
+
+        try {
+            PrintWriter pw = new PrintWriter(new FileOutputStream(fileName));
+            pw.write(autoComplete.toString());
+            pw.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
     private static String getType(ParameterDescription parameterDescription) {
         String type = "";
         if (parameterDescription.getParameter().arity() == 0) {

commons-lib/src/main/java/org/opencb/commons/utils/ListUtils.java

@@ -20,7 +20,7 @@
 import java.util.*;
 
 
-public class ListUtils {
+public class ListUtils extends CollectionUtils {
 
     public static <E> List<E> unique(List<E> array) {
         if (array == null) {

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>org.opencb.commons</groupId>
     <artifactId>commons</artifactId>
-    <version>3.5.1</version>
+    <version>3.6.0</version>
     <packaging>pom</packaging>
 
     <name>OpenCB commons project</name>
@@ -19,9 +19,9 @@
     </modules>
 
     <properties>
-        <java.commons.version>3.5.1</java.commons.version>
+        <java.commons.version>3.6.0</java.commons.version>
         <compileSource>1.8</compileSource>
-        <jackson.version>2.8.4</jackson.version>
+        <jackson.version>2.8.10</jackson.version>
         <apache.commons.version>3.5</apache.commons.version>
         <slf4j.version>1.7.21</slf4j.version>
         <avro.version>1.7.7</avro.version>