Supporting the option to drilldown in columns and bar charts

Application/src/main/resources/application.yml

@@ -7,6 +7,8 @@ spring:
     mvc:
         favicon:
             enabled: false
+    jackson:
+        default-property-inclusion: non_null
 
 management:
         endpoints:

Application/src/main/resources/static/js/CDF_chart_handler.js

@@ -144,6 +144,7 @@ function handleAdminSideData(dataJSONobj)
             //Pass the Chart library to ChartDataFormatter
             RequestInfoObj.library = dataJSONobj.library;
             RequestInfoObj.orderBy = dataJSONobj.orderBy;
+            RequestInfoObj.drilldown = dataJSONobj.drilldown;
             //Pass the Chart type to ChartDataFormatter
             var defaultType = dataJSONobj.chartDescription.chart.type;
             //Create ChartInfo Object Array
@@ -449,7 +450,9 @@ function convertToValidHighchartJson(responseData, originJson){
             seriesInstance.stacking = convertedJson.series[index].stacking;
 
         // Pass the data series name to the response data object
-        if(responseData.dataSeriesNames !== null)
+        //TODO check that this one did not break anything else!!!!!
+        // if(responseData.dataSeriesNames != null)
+        if(responseData.dataSeriesNames!== undefined && responseData.dataSeriesNames !== null)
             seriesInstance.name = responseData.dataSeriesNames[index];
 
         // Pass the data series type to the response data object
@@ -487,12 +490,16 @@ function convertToValidHighchartJson(responseData, originJson){
             if(convertedJson.xAxis === undefined)
                 convertedJson.xAxis = {};    
             convertedJson.xAxis.categories = responseData.xAxis_categories;
+
+            //TODO check that we do not need to use the default 'linear' type of xAxis
+            if (responseData.xAxis_categories === undefined && (seriesInstance.type === "column" || seriesInstance.type === "bar"))
+                convertedJson.xAxis.type = "category";
         }
 
         convertedJson.series[index] = seriesInstance;       
     }
 
-    if(responseData.drilldown !== null){
+    if(responseData.drilldown !== undefined && responseData.drilldown !== null){
         convertedJson.drilldown = new Object();
         convertedJson.drilldown.series = new Array( Object.keys(responseData.drilldown).length );
 
@@ -504,7 +511,8 @@ function convertToValidHighchartJson(responseData, originJson){
             convertedJson.drilldown.series[index].name = responseData.series[0].data[index].drilldown;
             // ! Hardcoded Selection that a drilldown is always a pie !
             // As of now drilldown is ONLY used in a pie-graph of a single series with a second group by
-            convertedJson.drilldown.series[index].type = "pie";
+            // convertedJson.drilldown.series[index].type = "pie";
+            convertedJson.drilldown.series[index].type = seriesInstance.type;
         }
     }
 

ChartDataFormatter/src/main/java/gr/uoa/di/madgik/ChartDataFormatter/DataFormatter/HighChartsDataFormatter.java

@@ -42,8 +42,12 @@ public HighChartsJsonResponse toJsonResponse(List<Result> dbAccessResults, Objec
         List<SupportedChartTypes> chartsType = (List<SupportedChartTypes>) args[0];
         List<String> chartNames = (List<String>) args[1];
 
+        //read isDrilldown field
+        boolean isDrilldown = (boolean) args[2];
+
+        //pass isDrilldown to singleToHighChartsJsonResponse
         if (dbAccessResults.size() == 1 && chartsType.size() == 1)
-            return singleToHighChartsJsonResponse(dbAccessResults.get(0), chartsType.get(0), chartNames.get(0));
+            return singleToHighChartsJsonResponse(dbAccessResults.get(0), chartsType.get(0), chartNames.get(0), isDrilldown);
 
         if (dbAccessResults.size() != chartsType.size())
             throw new DataFormationException("Result list and Chart Type list are of different size.");
@@ -59,7 +63,7 @@ private List<String> getXAxisCategories(List<Result> dbAccessResults) {
 
     private HighChartsJsonResponse singleToHighChartsJsonResponse(Result result,
                                                                   SupportedChartTypes chartType,
-                                                                  String chartName) throws DataFormationException {
+                                                                  String chartName, boolean isDrilldown) throws DataFormationException {
 
         //There are no Results
         if (result.getRows().isEmpty())
@@ -74,7 +78,8 @@ private HighChartsJsonResponse singleToHighChartsJsonResponse(Result result,
                 if (chartType == SupportedChartTypes.dependencywheel || chartType == SupportedChartTypes.sankey)
                     return HCGraph(result, false, chartType, chartName);
 
-                return HCDoubleGroupBy(result, chartType);
+                //handling isDrilldown in DoubleGroupBy
+                return HCDoubleGroupBy(result, chartType, isDrilldown);
             case 4:
                 if (chartType == SupportedChartTypes.dependencywheel || chartType == SupportedChartTypes.sankey)
                     return HCGraph(result, true, chartType, chartName);
@@ -139,18 +144,31 @@ private HighChartsJsonResponse HCSingleGroupBy(Result result, SupportedChartType
         return new HighChartsJsonResponse(dataSeries, new ArrayList<>(xAxis_categories.keySet()), chartNames, chartTypes);
     }
 
-    private HighChartsJsonResponse HCDoubleGroupBy(Result result, SupportedChartTypes chartType) {
+    private HighChartsJsonResponse HCDoubleGroupBy(Result result, SupportedChartTypes chartType, boolean isDrilldown) {
+
+        System.out.println("Result: " + result);
 
         LinkedHashMap<String, Integer> xAxis_categories = new LinkedHashMap<>();
         LinkedHashMap<String, HashMap<String, String>> groupByMap = new LinkedHashMap<>();
         ArrayList<AbsData> dataSeries = new ArrayList<>();
         ArrayList<String> dataSeriesTypes = new ArrayList<>();
 
         for (List<?> row : result.getRows()) {
+            String groupByValue;
+            String xValueA;
 
-            // Create a map with the unique values for the group by
-            String groupByValue = String.valueOf(row.get(2));
-            String xValueA = String.valueOf(row.get(1));
+            if (chartType.equals(SupportedChartTypes.pie)) {
+                groupByValue = String.valueOf(row.get(1));
+                xValueA = String.valueOf(row.get(2));
+            } else {
+                if (!isDrilldown) {
+                    groupByValue = String.valueOf(row.get(2));
+                    xValueA = String.valueOf(row.get(1));
+                } else {
+                    groupByValue = String.valueOf(row.get(1));
+                    xValueA = String.valueOf(row.get(2));
+                }
+            }
             // I assume that always the first value of the row is for the Y value
             String yValue = String.valueOf(row.get(0));
 
@@ -166,74 +184,81 @@ private HighChartsJsonResponse HCDoubleGroupBy(Result result, SupportedChartType
 
         switch (chartType) {
             case area:
-            case bar:
-            case column:
             case line:
             case treemap:
+                return HCDoubleGroupByNoDrilldown(chartType, xAxis_categories, groupByMap, dataSeries, dataSeriesTypes);
+            case bar:
+            case column:
+                if (isDrilldown)
+                    return HCDoubleGroupByDrilldown(chartType, xAxis_categories, groupByMap, dataSeries, dataSeriesTypes);
+                else
+                    return HCDoubleGroupByNoDrilldown(chartType, xAxis_categories, groupByMap, dataSeries, dataSeriesTypes);
+            case pie:
+                return HCDoubleGroupByDrilldown(chartType, xAxis_categories, groupByMap, dataSeries, dataSeriesTypes);
+            default:
+                return null;
+        }
+    }
 
-                for (HashMap<String, String> XValueToYValueMapping : groupByMap.values()) {
-
-                    ArrayList<Number> yValuesArray = new ArrayList<>();
-                    for (String xValue : xAxis_categories.keySet()) {
-
-                        if (XValueToYValueMapping.containsKey(xValue)) {
-
-                            String yValue = XValueToYValueMapping.get(xValue);
-                            yValuesArray.add(parseValue(yValue));
-
-                        } else
-                            yValuesArray.add(null);
-                    }
-                    dataSeries.add(new ArrayOfValues(yValuesArray));
-                    dataSeriesTypes.add(chartType.name());
-                }
+    private HighChartsJsonResponse HCDoubleGroupByDrilldown(SupportedChartTypes chartType, LinkedHashMap<String, Integer> xAxis_categories, LinkedHashMap<String, HashMap<String, String>> groupByMap, ArrayList<AbsData> dataSeries, ArrayList<String> dataSeriesTypes) {
+        ArrayList<DataObject> mainSlicesValuesArray = new ArrayList<>();
+        ArrayList<AbsData> drillDownArray = new ArrayList<>();
 
-                return new HighChartsJsonResponse(dataSeries, new ArrayList<>(xAxis_categories.keySet()),
-                        new ArrayList<>(groupByMap.keySet()), dataSeriesTypes);
+        for (String groupByX : groupByMap.keySet()) {
 
-            case pie:
+            HashMap<String, String> XValueToYValueMapping = groupByMap.get(groupByX);
 
-                ArrayList<DataObject> mainSlicesValuesArray = new ArrayList<>();
-                ArrayList<AbsData> drillDownArray = new ArrayList<>();
+            float pieSliceSum = 0;
+            ArrayList<DataObject> drillDownSliceValuesArray = new ArrayList<>();
 
-                for (String groupByX : groupByMap.keySet()) {
+            for (String xValue : xAxis_categories.keySet()) {
 
-                    HashMap<String, String> XValueToYValueMapping = groupByMap.get(groupByX);
+                String yValue = XValueToYValueMapping.get(xValue);
+                Number value = parseValue(yValue);
+                drillDownSliceValuesArray.add(new DataObject(xValue, value));
+                if (value != null) {
+                    pieSliceSum += value.floatValue();
+                }
 
-                    float pieSliceSum = 0;
-                    ArrayList<DataObject> drillDownSliceValuesArray = new ArrayList<>();
+            }
+            drillDownArray.add(new ArrayOfDataObjects(drillDownSliceValuesArray));
 
-                    for (String xValue : xAxis_categories.keySet()) {
+            DataObject pieSlice = new DataObject(groupByX, pieSliceSum);
+            pieSlice.setDrilldown(groupByX);
 
-                        String yValue = XValueToYValueMapping.get(xValue);
-                        Number value = parseValue(yValue);
-                        drillDownSliceValuesArray.add(new DataObject(xValue, value));
-                        if (value != null) {
-                            pieSliceSum += value.floatValue();
-                        }
+            mainSlicesValuesArray.add(pieSlice);
+        }
 
-                    }
-                    drillDownArray.add(new ArrayOfDataObjects(drillDownSliceValuesArray));
+        dataSeries.add(new ArrayOfDataObjects(mainSlicesValuesArray));
+        dataSeriesTypes.add(chartType.name());
+        log.debug("Added " + chartType.name());
 
-                    DataObject pieSlice = new DataObject(groupByX, pieSliceSum);
-                    pieSlice.setDrilldown(groupByX);
+        //HighChartsJsonResponse ret = new HighChartsJsonResponse(dataSeries, new ArrayList<>(xAxis_categories.keySet()), null, dataSeriesTypes);
+        HighChartsJsonResponse ret = new HighChartsJsonResponse(dataSeries, null, null, dataSeriesTypes);
+        ret.setDrilldown(drillDownArray);
+        return ret;
+    }
 
-                    mainSlicesValuesArray.add(pieSlice);
-                }
+    private HighChartsJsonResponse HCDoubleGroupByNoDrilldown(SupportedChartTypes chartType, LinkedHashMap<String, Integer> xAxis_categories, LinkedHashMap<String, HashMap<String, String>> groupByMap, ArrayList<AbsData> dataSeries, ArrayList<String> dataSeriesTypes) {
+        for (HashMap<String, String> XValueToYValueMapping : groupByMap.values()) {
 
-                dataSeries.add(new ArrayOfDataObjects(mainSlicesValuesArray));
-                dataSeriesTypes.add(chartType.name());
-                log.debug("Added " + chartType.name());
+            ArrayList<Number> yValuesArray = new ArrayList<>();
+            for (String xValue : xAxis_categories.keySet()) {
 
-                HighChartsJsonResponse ret = new HighChartsJsonResponse(dataSeries, new ArrayList<>(xAxis_categories.keySet()),
-                        null, dataSeriesTypes);
-                ret.setDrilldown(drillDownArray);
-                return ret;
+                if (XValueToYValueMapping.containsKey(xValue)) {
 
+                    String yValue = XValueToYValueMapping.get(xValue);
+                    yValuesArray.add(parseValue(yValue));
 
-            default:
-                return null;
+                } else
+                    yValuesArray.add(null);
+            }
+            dataSeries.add(new ArrayOfValues(yValuesArray));
+            dataSeriesTypes.add(chartType.name());
         }
+
+        return new HighChartsJsonResponse(dataSeries, new ArrayList<>(xAxis_categories.keySet()),
+                new ArrayList<>(groupByMap.keySet()), dataSeriesTypes);
     }
 
     private HighChartsJsonResponse multiToHighChartsJsonResponse(List<Result> dbAccessResults, List<SupportedChartTypes> chartsType, List<String> chartNames) throws DataFormationException {

ChartDataFormatter/src/main/java/gr/uoa/di/madgik/ChartDataFormatter/Handlers/RequestBodyHandler.java

@@ -50,7 +50,7 @@ public JsonResponse handleRequest(RequestInfo requestJson) throws RequestBodyExc
                     HighChartsJsonResponse highChartsJsonResponse;
                     try {
                         highChartsJsonResponse = new HighChartsDataFormatter().toJsonResponse(statsServiceResults,
-                                requestJson.getChartTypes(), requestJson.getChartNames());
+                                requestJson.getChartTypes(), requestJson.getChartNames(), requestJson.isDrilldown());
 
                     }catch (DataFormatter.DataFormationException e){
                         throw new RequestBodyException(e.getMessage(),e,HttpStatus.UNPROCESSABLE_ENTITY);

ChartDataFormatter/src/main/java/gr/uoa/di/madgik/ChartDataFormatter/JsonRepresentation/RequestBody/RequestInfo.java

@@ -19,13 +19,16 @@ public class RequestInfo {
     private List<ChartInfo> chartsInfo;
     @JsonProperty
     private String orderBy;
+    @JsonProperty
+    private boolean drilldown;
 
     public RequestInfo() {}
 
-    public RequestInfo(String library, List<ChartInfo> chartsInfo, String orderBy) {
+    public RequestInfo(String library, List<ChartInfo> chartsInfo, String orderBy, boolean drilldown) {
         this.library = library;
         this.chartsInfo = chartsInfo;
         this.orderBy = orderBy;
+        this.drilldown = drilldown;
     }
 
     public String getLibrary() {
@@ -48,6 +51,14 @@ public void setOrderBy(String orderBy) {
         this.orderBy = orderBy;
     }
 
+    public boolean isDrilldown() {
+        return drilldown;
+    }
+
+    public void setDrilldown(boolean drilldown) {
+        this.drilldown = drilldown;
+    }
+
     public List<SupportedChartTypes> getChartTypes(){
 
         ArrayList<SupportedChartTypes> retList = new ArrayList<>();

DBAccess/src/main/resources/application.yml

@@ -1,41 +1,27 @@
 spring:
   datasources:
-    - id: openaire.public
-      username:
-      password:
-      url: jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/openaire_prod_stats;UseNativeQuery=1
-      driver-class-name: com.cloudera.impala.jdbc41.Driver
-    - id: openaire.shadow
-      username:
-      password:
-      url: jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/openaire_prod_stats_shadow;UseNativeQuery=1
-      driver-class-name: com.cloudera.impala.jdbc41.Driver
     - id: monitor.public
-      username:
-      password:
-      url: jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/openaire_prod_stats_monitor;UseNativeQuery=1
-      driver-class-name: com.cloudera.impala.jdbc41.Driver
-    - id: monitor.public
-      username:
-      password:
-      url: jdbc:impala://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/openaire_prod_stats_monitor_shadow;UseNativeQuery=1
-      driver-class-name: com.cloudera.impala.jdbc41.Driver
+      url: jdbc:postgresql://esperos.di.uoa.gr:5432/monitor
+      driver-class-name: org.postgresql.Driver
+      username: dnet
+      password: dnetPwd
     - id: cache
       username: sa
       password:
       url: jdbc:hsqldb:file:/tmp/cache
       driver-class-name: org.hsqldb.jdbcDriver
   redis:
-    host: localhost
+    host: vereniki.athenarc.gr
     port: 6379
+    password: redisPassword
   # Disable feature detection by this undocumented parameter. Check the org.hibernate.engine.jdbc.internal.JdbcServiceImpl.configure method for more details.
   jpa:
+    database-platform: org.hibernate.dialect.PostgreSQLDialect
     properties:
       hibernate:
         temp:
           use_jdbc_metadata_defaults: false
-  # Because detection is disabled you have to set correct dialect by hand.
-    database-platform: org.hibernate.dialect.PostgreSQL9Dialect
+    database: postgresql
   datasource:
     hikari:
       connectionTimeout: 20000
@@ -45,10 +31,13 @@ spring:
 statstool:
   result_limit: 70
   cache:
-    enabled: true
+    enabled: false
     storage: redis
     update:
       entries: 5000
       time: 10800
   namedqueries:
     path: ...
+  mappings:
+    file:
+      path: classpath:mappings.json

DBAccess/src/main/resources/mappings.json

@@ -10,14 +10,14 @@
     "file": "classpath:mapping.json"
   },
   {
-    "name": "OpenAIRE original",
+    "name": "monitor",
     "description": "Contains all the OpenAIRE information space but its entities correspond to the original OpenAIRE schema",
     "usage": "Best used to navigate the whole information space",
     "shareholders": ["Admins", "Moderators"],
     "complexity": 0,
     "primary": false,
     "hidden": false,
-    "file": "classpath:openaire_db.json"
+    "file": "classpath:monitor.json"
   },
   {
     "name": "Stats db",