From ebc7d3b117745ae296298ace3f651042d8043702 Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 18 Nov 2019 17:38:06 +0530 Subject: [PATCH 001/152] Initial Commit for DSS Ingest Pipeline and Analytics --- dashboard-analytics/pom.xml | 145 ++ .../com/tarento/analytics/AnalyticApp.java | 35 + .../analytics/ConfigurationLoader.java | 92 ++ .../tarento/analytics/constant/Constants.java | 92 ++ .../constant/ElasticSearchConstants.java | 22 + .../tarento/analytics/constant/ErrorCode.java | 10 + .../controllers/DashboardController.java | 125 ++ .../analytics/dao/ElasticSearchDao.java | 41 + .../dao/impl/ElasticSearchDaoImpl.java | 1280 +++++++++++++++++ .../tarento/analytics/dto/AggregateDto.java | 85 ++ .../analytics/dto/AggregateRequestDto.java | 77 + .../analytics/dto/AggregateRequestDtoV2.java | 74 + .../dto/CummulativeDataRequestDto.java | 32 + .../tarento/analytics/dto/DashboardDto.java | 44 + .../analytics/dto/DashboardHeaderDto.java | 28 + .../java/com/tarento/analytics/dto/Data.java | 57 + .../com/tarento/analytics/dto/MappingDto.java | 14 + .../java/com/tarento/analytics/dto/Plot.java | 40 + .../tarento/analytics/dto/RequestDate.java | 26 + .../tarento/analytics/dto/RequestDtoV2.java | 22 + .../tarento/analytics/dto/ResponseDto.java | 40 + .../com/tarento/analytics/dto/RoleDto.java | 46 + .../com/tarento/analytics/dto/SearchDto.java | 165 +++ .../com/tarento/analytics/dto/UserDto.java | 69 + .../tarento/analytics/enums/ChartType.java | 31 + .../analytics/exception/AINException.java | 29 + .../analytics/handler/IResponseHandler.java | 149 ++ .../handler/LineChartResponseHandler.java | 132 ++ .../handler/MetricChartResponseHandler.java | 73 + .../PerformanceChartResponeHandler.java | 118 ++ .../handler/PieChartResponseHandler.java | 78 + .../handler/ResponseHandlerFactory.java | 44 + .../handler/TableChartResponseHandler.java | 102 ++ .../model/ElasticSearchDictator.java | 69 + .../com/tarento/analytics/model/Item.java | 92 ++ .../com/tarento/analytics/model/KeyData.java | 53 + .../com/tarento/analytics/model/Label.java | 19 + .../com/tarento/analytics/model/LineData.java | 20 + .../com/tarento/analytics/model/Query.java | 26 + .../tarento/analytics/model/ServiceQuery.java | 27 + .../tarento/analytics/model/Transaction.java | 264 ++++ .../model/dashboardConfig/Chart.java | 108 ++ .../model/dashboardConfig/Dashboard.java | 100 ++ .../model/dashboardConfig/Header.java | 55 + .../model/dashboardConfig/Visualization.java | 114 ++ .../analytics/org/service/ClientService.java | 14 + .../org/service/TarentoServiceImpl.java | 95 ++ .../analytics/query/model/Aggregation.java | 5 + .../tarento/analytics/query/model/Query.java | 28 + .../tarento/analytics/query/model/XAxis.java | 18 + .../tarento/analytics/query/model/YAxis.java | 17 + .../repository/ElasticSearchRepository.java | 84 ++ .../analytics/service/MetadataService.java | 13 + .../analytics/service/QueryService.java | 30 + .../service/impl/MetadataServiceImpl.java | 64 + .../service/impl/QueryServiceImpl.java | 457 ++++++ .../service/impl/QueryServiceTemplate.java | 95 ++ .../analytics/utils/ElasticProperties.java | 73 + .../analytics/utils/ElasticSearchClient.java | 48 + .../analytics/utils/JSONObjectUtil.java | 41 + .../com/tarento/analytics/utils/JsonKey.java | 20 + .../tarento/analytics/utils/PathRoutes.java | 41 + .../tarento/analytics/utils/ResponseCode.java | 87 ++ .../analytics/utils/ResponseGenerator.java | 86 ++ .../src/main/resources/application.properties | 55 + .../main/resources/application.properties.j2 | 75 + .../main/resources/schema/ChartApiConfig.json | 725 ++++++++++ .../resources/schema/RoleDashboardConfig.json | 560 ++++++++ .../src/main/resources/schema/getchart.json | 43 + .../src/main/resources/schema/query.json | 20 + .../src/main/resources/schema/query.yaml | 9 + dashboard-ingest/pom.xml | 195 +++ .../com/ingestpipeline/AppStartupRunner.java | 42 + .../java/com/ingestpipeline/IngestApp.java | 37 + .../config/ElasticSearchConfiguration.java | 66 + .../consumer/ConsumerConfigurations.java | 127 ++ .../consumer/EnrichmentConsumer.java | 45 + .../consumer/HashMapDeserializer.java | 14 + .../consumer/KafkaConsumer.java | 15 + .../consumer/TransformConsumer.java | 48 + .../consumer/ValidatorConsumer.java | 64 + .../controller/RestApiController.java | 201 +++ .../model/CollectionDomainConfig.java | 83 ++ .../ingestpipeline/model/CollectionRef.java | 38 + .../com/ingestpipeline/model/Consumable.java | 58 + .../model/DomainIndexConfig.java | 71 + .../ingestpipeline/model/IncomingData.java | 35 + .../model/IncomingDataConfig.java | 45 + .../ingestpipeline/model/KeyValuePair.java | 72 + .../com/ingestpipeline/model/Snippet.java | 8 + .../com/ingestpipeline/model/TargetData.java | 148 ++ .../producer/HashMapSerializer.java | 33 + .../producer/IngestProducer.java | 24 + .../producer/IngestProducerConfig.java | 67 + .../producer/JavaSerializer.java | 37 + .../repository/ElasticSearchRepository.java | 58 + .../repository/TargetDataDao.java | 18 + .../service/ElasticService.java | 233 +++ .../service/EnrichTransform.java | 51 + .../service/EnrichmentService.java | 14 + .../service/EnrichmentServiceImpl.java | 201 +++ .../ingestpipeline/service/IESService.java | 111 ++ .../ingestpipeline/service/IngestService.java | 14 + .../service/IngestServiceImpl.java | 57 + .../service/TransformService.java | 14 + .../service/TransformServiceImpl.java | 46 + .../service/ValidationService.java | 14 + .../service/ValidationServiceImpl.java | 73 + .../util/ApplicationProperties.java | 162 +++ .../com/ingestpipeline/util/ConfigLoader.java | 84 ++ .../com/ingestpipeline/util/Constants.java | 66 + .../ingestpipeline/util/CustomErrorType.java | 16 + .../com/ingestpipeline/util/JSONUtil.java | 94 ++ .../com/ingestpipeline/util/JSONUtils.java | 16 + .../java/com/ingestpipeline/util/JsonKey.java | 20 + .../util/MockMultipartFile.java | 77 + .../com/ingestpipeline/util/ReadUtil.java | 347 +++++ .../com/ingestpipeline/util/ResponseCode.java | 85 ++ .../util/ResponseGenerator.java | 79 + .../ingestpipeline/util/ResponseMessages.java | 11 + .../com/ingestpipeline/util/ResponseUtil.java | 91 ++ .../src/main/resources/application.properties | 69 + .../config/CollectionDomainConfig.json | 51 + .../main/resources/config/config.properties | 18 + .../config/enrichment_transaction_v1.json | 49 + .../config/transform_collection_v1.json | 102 ++ .../resources/config/transform_pgr_v1.json | 12 + .../resources/config/transform_pt_v1.json | 10 + .../resources/config/transform_target_v1.json | 20 + .../resources/config/transform_tl_v1.json | 27 + .../config/transform_transaction_v1.json | 29 + .../config/validator_transaction_v1.json | 32 + dashboard-ingest/src/main/resources/data.sql | 13 + .../testcases/IngestTestClient.java | 54 + 134 files changed, 11278 insertions(+) create mode 100644 dashboard-analytics/pom.xml create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/AnalyticApp.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/ConfigurationLoader.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/constant/Constants.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/constant/ElasticSearchConstants.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/constant/ErrorCode.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/controllers/DashboardController.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dao/ElasticSearchDao.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dao/impl/ElasticSearchDaoImpl.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDtoV2.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/CummulativeDataRequestDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardHeaderDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/Data.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/MappingDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/Plot.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDate.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDtoV2.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/ResponseDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/RoleDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/SearchDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/UserDto.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/enums/ChartType.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/exception/AINException.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/handler/IResponseHandler.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/handler/LineChartResponseHandler.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricChartResponseHandler.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/handler/PerformanceChartResponeHandler.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/handler/PieChartResponseHandler.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/handler/ResponseHandlerFactory.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/handler/TableChartResponseHandler.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/ElasticSearchDictator.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/Item.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/KeyData.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/Label.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/LineData.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/Query.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/ServiceQuery.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/Transaction.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Chart.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Dashboard.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Header.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Visualization.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientService.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/org/service/TarentoServiceImpl.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Aggregation.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Query.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/query/model/XAxis.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/query/model/YAxis.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/repository/ElasticSearchRepository.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/service/MetadataService.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/service/QueryService.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/MetadataServiceImpl.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceImpl.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceTemplate.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticProperties.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticSearchClient.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/utils/JSONObjectUtil.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/utils/JsonKey.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/utils/PathRoutes.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseCode.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseGenerator.java create mode 100644 dashboard-analytics/src/main/resources/application.properties create mode 100644 dashboard-analytics/src/main/resources/application.properties.j2 create mode 100644 dashboard-analytics/src/main/resources/schema/ChartApiConfig.json create mode 100644 dashboard-analytics/src/main/resources/schema/RoleDashboardConfig.json create mode 100644 dashboard-analytics/src/main/resources/schema/getchart.json create mode 100644 dashboard-analytics/src/main/resources/schema/query.json create mode 100644 dashboard-analytics/src/main/resources/schema/query.yaml create mode 100644 dashboard-ingest/pom.xml create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/AppStartupRunner.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/IngestApp.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/config/ElasticSearchConfiguration.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ConsumerConfigurations.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/consumer/EnrichmentConsumer.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/consumer/HashMapDeserializer.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/consumer/KafkaConsumer.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/consumer/TransformConsumer.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ValidatorConsumer.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/controller/RestApiController.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/model/CollectionDomainConfig.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/model/CollectionRef.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/model/Consumable.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/model/DomainIndexConfig.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingData.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingDataConfig.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/model/KeyValuePair.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/model/Snippet.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetData.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/producer/HashMapSerializer.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducer.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducerConfig.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/producer/JavaSerializer.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/repository/ElasticSearchRepository.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/repository/TargetDataDao.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/ElasticService.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichTransform.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentService.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentServiceImpl.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/IESService.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestService.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestServiceImpl.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformService.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformServiceImpl.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationService.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationServiceImpl.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/ApplicationProperties.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/ConfigLoader.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/Constants.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/CustomErrorType.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtil.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtils.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/JsonKey.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/MockMultipartFile.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/ReadUtil.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseCode.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseGenerator.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseMessages.java create mode 100644 dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseUtil.java create mode 100644 dashboard-ingest/src/main/resources/application.properties create mode 100644 dashboard-ingest/src/main/resources/config/CollectionDomainConfig.json create mode 100644 dashboard-ingest/src/main/resources/config/config.properties create mode 100644 dashboard-ingest/src/main/resources/config/enrichment_transaction_v1.json create mode 100644 dashboard-ingest/src/main/resources/config/transform_collection_v1.json create mode 100644 dashboard-ingest/src/main/resources/config/transform_pgr_v1.json create mode 100644 dashboard-ingest/src/main/resources/config/transform_pt_v1.json create mode 100644 dashboard-ingest/src/main/resources/config/transform_target_v1.json create mode 100644 dashboard-ingest/src/main/resources/config/transform_tl_v1.json create mode 100644 dashboard-ingest/src/main/resources/config/transform_transaction_v1.json create mode 100644 dashboard-ingest/src/main/resources/config/validator_transaction_v1.json create mode 100644 dashboard-ingest/src/main/resources/data.sql create mode 100644 dashboard-ingest/src/test/java/com/ingestpipeline/testcases/IngestTestClient.java diff --git a/dashboard-analytics/pom.xml b/dashboard-analytics/pom.xml new file mode 100644 index 000000000..69625fba9 --- /dev/null +++ b/dashboard-analytics/pom.xml @@ -0,0 +1,145 @@ + + 4.0.0 + com.tarento + analytics + 0.0.1-SNAPSHOT + + Analytic Service + Project for handling the data for analytics visualization + http://maven.apache.org + + org.springframework.boot + spring-boot-starter-parent + 1.5.3.RELEASE + + + + UTF-8 + UTF-8 + 1.8 + 3.0.1 + + + + + junit + junit + 3.8.1 + test + + + mysql + mysql-connector-java + 5.1.21 + + + org.springframework.kafka + spring-kafka + 1.1.2.RELEASE + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-jdbc + 2.0.0.RELEASE + + + + com.google.code.gson + gson + 2.8.0 + + + + org.springframework.boot + spring-boot-starter-jersey + 1.2.0.RELEASE + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + + + org.projectlombok + lombok + + + org.apache.poi + poi-ooxml + 3.10-FINAL + + + + javax.ws.rs + jsr311-api + 1.1.1 + + + org.apache.cxf + cxf-rt-frontend-jaxws + ${cxf.version} + + + org.apache.cxf + cxf-rt-transports-http + ${cxf.version} + + + org.apache.cxf + cxf-rt-frontend-jaxrs + ${cxf.version} + + + org.apache.cxf + cxf-tools-java2ws + ${cxf.version} + + + org.apache.commons + commons-lang3 + 3.4 + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.2.4 + + + org.elasticsearch + elasticsearch + 6.2.4 + + + com.github.fge + json-schema-validator + 2.2.6 + + + + commons-io + commons-io + 2.6 + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/AnalyticApp.java b/dashboard-analytics/src/main/java/com/tarento/analytics/AnalyticApp.java new file mode 100644 index 000000000..883def0f5 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/AnalyticApp.java @@ -0,0 +1,35 @@ +package com.tarento.analytics; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; + +import com.tarento.analytics.constant.Constants; + + +@SpringBootApplication +public class AnalyticApp { + public static void main( String[] args ) { + SpringApplication.run(AnalyticApp.class, args); + } + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + + @Bean + public WebMvcConfigurer corsConfigurer() { + return new WebMvcConfigurerAdapter() { + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/**").allowedMethods(Constants.GET, Constants.POST,Constants.PUT, Constants.DELETE, Constants.OPTIONS).allowedOrigins("*") + .allowedHeaders("*"); + } + }; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/ConfigurationLoader.java b/dashboard-analytics/src/main/java/com/tarento/analytics/ConfigurationLoader.java new file mode 100644 index 000000000..b913760f3 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/ConfigurationLoader.java @@ -0,0 +1,92 @@ +package com.tarento.analytics; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.core.io.support.ResourcePatternUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Map; + +@Component("configurationLoader") +public class ConfigurationLoader { + + private static Logger logger = LoggerFactory.getLogger(ConfigurationLoader.class); + private Map nameContentMap = new HashMap<>(); + @Autowired + private ResourceLoader resourceLoader; + @Autowired + private ObjectMapper objectMapper; + private static final String RESOURCE_LOCATION = "classpath*:schema/*.json"; + public static final String ROLE_DASHBOARD_CONFIG = "RoleDashboardConfig.json"; + + + + /** + * Loads config resources + * @throws Exception + */ + @PostConstruct + public void loadResources() throws Exception { + Resource[] resources = getResources(RESOURCE_LOCATION); + + for (Resource resource : resources) { + String jsonContent = getContent(resource); + ObjectNode jsonNode = (ObjectNode) objectMapper.readTree(jsonContent); + nameContentMap.put(resource.getFilename(), jsonNode); + } + logger.info("Number of resources loaded " + nameContentMap.size()); + + } + + /** + * Obtains a ObjectNode w.r.t given resource/file name in classpath*:schema + * @param name + * @return + */ + public ObjectNode get(String name) { + return nameContentMap.get(name); + } + + /** + * Loads all the resources/files with a given pattern *.json + * @param pattern path with *json + * @return + * @throws IOException + */ + private Resource[] getResources(String pattern) throws IOException { + Resource[] resources = ResourcePatternUtils.getResourcePatternResolver(resourceLoader).getResources(pattern); + return resources; + } + + /** + * Returns a content of resource + * + * @param resource + * @return + */ + private String getContent(Resource resource) { + String content = null; + try { + InputStream is = resource.getInputStream(); + byte[] encoded = IOUtils.toByteArray(is); + content = new String(encoded, Charset.forName("UTF-8")); + + } catch (IOException e) { + logger.error("Cannot load resource " + resource.getFilename()); + + } + return content; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/constant/Constants.java b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/Constants.java new file mode 100644 index 000000000..ccd1dcdcd --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/Constants.java @@ -0,0 +1,92 @@ +package com.tarento.analytics.constant; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class Constants { + /** + * Allowed Origins for CORS Bean + */ + public static final String GET = "GET"; + public static final String POST = "POST"; + public static final String PUT = "PUT"; + public static final String DELETE = "DELETE"; + public static final String OPTIONS = "OPTIONS"; + + + public static int UNAUTHORIZED_ID = 401; + public static int SUCCESS_ID = 200; + public static int FAILURE_ID = 320; + public static String UNAUTHORIZED = "Invalid credentials. Please try again."; + public static String PROCESS_FAIL = "Process failed, Please try again."; + public static String SUCCESS= "success"; + + + //chart format + + public static final String D3 = "d3"; + public static final String CHARTJS = "chartjs"; + + //chart type + public static final String BAR = "bar"; + public static final String PIE ="pie"; + public static final String STACKEDBAR ="stackedbar"; + public static final String LINE ="line"; + public static final String HORIZONTAL_BAR="horizontalBar"; + public static final String DOUGHNUT="doughnut"; + public static final String Heat = "heat"; + public static final String RADAR ="radar"; + + public static final Long FEEDBACK_MESSAGE_TIMEOUT = 2000l; + + public static final String STORE_ID = "storeId"; + + public static final String PLACEMENTS_DASHBOARD = "DASHBOARD"; + public static final String PLACEMENTS_HOME = "HOME"; + + public static final List RATING_LIST = new ArrayList<>(Arrays.asList(1l,2l,3l,4l,5l)); + public static final List RATING_LIST_STRING = new ArrayList<>(Arrays.asList("1","2","3","4","5")); + public static final List RATING_LIST_STRING_STAR = new ArrayList<>(Arrays.asList("1 Star","2 Star","3 Star","4 Star","5 Star")); + + public interface Modules { + final static String HOME_REVENUE = "HOME_REVENUE"; + final static String HOME_SERVICES = "HOME_SERVICES"; + final static String COMMON = "COMMON"; + final static String PT = "PT"; + final static String TL = "TL"; + } + + public interface KafkaTopics { + final static String NEW_CONTENT_MESSAGE = "SaveContent"; + final static String SIMULATOR_TRANSACTION = "SaveTransaction"; + } + + public interface ConfigurationFiles { + final static String CHART_API_CONFIG = "ChartApiConfig.json"; + } + + public interface JsonPaths { + final static String CHART_TYPE = "chartType"; + final static String QUERIES = "queries"; + final static String AGGREGATION_QUERY= "aggrQuery"; + final static String INDEX_NAME = "indexName"; + final static String REQUEST_QUERY_MAP = "requestQueryMap"; + final static String DATE_REF_FIELD = "dateRefField"; + final static String AGGS = "aggs"; + final static String AGGREGATIONS = "aggregations" ; + final static String MODULE = "module"; + } + + public interface Filters { + final static String MODULE = "module"; + final static String FILTER_ALL = "*"; + } + + public interface Catagory { + final static String SEVICE = "service"; + final static String REVENUE = "revenue"; + } + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ElasticSearchConstants.java b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ElasticSearchConstants.java new file mode 100644 index 000000000..09963b4dc --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ElasticSearchConstants.java @@ -0,0 +1,22 @@ +package com.tarento.analytics.constant; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ElasticSearchConstants { + public static final String LTE = "<="; + public static final String LT = "<"; + public static final String GTE = ">="; + public static final String GT = ">"; + public static final String ASC_ORDER = "ASC"; + public static final String STARTS_WITH = "startsWith"; + public static final String ENDS_WITH = "endsWith"; + public static final List upsertResults = + new ArrayList<>(Arrays.asList("CREATED", "UPDATED", "NOOP")); + public static final String SOFT_MODE = "soft"; + public static final String RAW_APPEND = ".raw"; + public static final String DAY_OF_WEEK = "dayOfWeek"; + public static final String DAY = "day"; + public static final String HOUR = "hour"; +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ErrorCode.java b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ErrorCode.java new file mode 100644 index 000000000..56a8e5cc4 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/constant/ErrorCode.java @@ -0,0 +1,10 @@ +package com.tarento.analytics.constant; + +public class ErrorCode { + + public static final String ERR320= "320"; + + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/controllers/DashboardController.java b/dashboard-analytics/src/main/java/com/tarento/analytics/controllers/DashboardController.java new file mode 100644 index 000000000..20fd16dfa --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/controllers/DashboardController.java @@ -0,0 +1,125 @@ +package com.tarento.analytics.controllers; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.tarento.analytics.dto.*; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.request.ServletWebRequest; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.constant.ErrorCode; +import com.tarento.analytics.exception.AINException; +import com.tarento.analytics.org.service.ClientService; +import com.tarento.analytics.service.MetadataService; +import com.tarento.analytics.utils.PathRoutes; +import com.tarento.analytics.utils.ResponseGenerator; + +@RestController +@RequestMapping(PathRoutes.DashboardApi.DASHBOARD_ROOT_PATH) +public class DashboardController { + + public static final Logger logger = LoggerFactory.getLogger(DashboardController.class); + + @Autowired + private MetadataService metadataService; + + + @Autowired + private ClientService clientService; + + @GetMapping(value = PathRoutes.DashboardApi.TEST_PATH, produces = MediaType.APPLICATION_JSON_VALUE) + public String getTest() throws JsonProcessingException { + return ResponseGenerator.successResponse("success"); + + } + + //TODO: intgrate with user Auth + @RequestMapping(value = PathRoutes.DashboardApi.GET_CHART_V2, method = RequestMethod.POST) + public ResponseEntity getVisualizationChartV2(@RequestBody RequestDtoV2 requestDto, @RequestHeader(value = "x-user-info", required = false) String xUserInfo, ServletWebRequest request) { + + /*logger.info("Request Detail:" + requestDto); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + UserDto user = gson.fromJson(xUserInfo, UserDto.class);*/ + + // TODO: move to userAuth helper + UserDto user = new UserDto(); + logger.info("user"+xUserInfo); + + AggregateRequestDtoV2 requestInfo = requestDto.getAggregationRequestDto(); + Map headers = requestDto.getHeaders(); + //requestInfo.getFilters().putAll(headers); + ResponseEntity responseEntity = null; + try { + if (headers.isEmpty() || headers.get("tenantId") == null) { + logger.error("Please provide header details"); + responseEntity = new ResponseEntity(new ResponseDto(HttpStatus.BAD_REQUEST.value(), "header or tenantId is missing", null), HttpStatus.BAD_REQUEST); + + } else { + // To be removed once the development is complete + if(StringUtils.isBlank(requestInfo.getModuleLevel())) { + requestInfo.setModuleLevel(Constants.Modules.HOME_REVENUE); + } + + AggregateDto responseData = clientService.getAggregatedData(requestInfo, user.getRoles()); + responseEntity = new ResponseEntity(new ResponseDto(HttpStatus.OK.value(), "success", responseData), HttpStatus.OK); + } + + } catch (Exception e) { + logger.error("error while executing api getVisualizationChart", e.getMessage()); + responseEntity = new ResponseEntity(new ResponseDto(HttpStatus.INTERNAL_SERVER_ERROR.value(), e.getMessage(), null), HttpStatus.INTERNAL_SERVER_ERROR); + } + return responseEntity; + } + + //TODO: integrate with user Auth + @RequestMapping(value = PathRoutes.DashboardApi.GET_DASHBOARD_CONFIG + "/{dashboardId}", method = RequestMethod.GET) + public ResponseEntity getDashboardConfiguration(@PathVariable String dashboardId, @RequestParam(value="catagory", required = false) String catagory, @RequestHeader(value = "x-user-info", required = false) String xUserInfo) { + + //Gson gson = new GsonBuilder().setPrettyPrinting().create(); + //TODO- to remove the hard coded stuffs once integrated and comes from + + try { + UserDto user = new UserDto(); + user.setId(new Long("10007")); + user.setOrgId("1"); + user.setCountryCode(""); + RoleDto role = new RoleDto(); + role.setId(new Long("6")); + role.setName("HR User"); + List roles = new ArrayList<>(); + roles.add(role); + user.setRoles(roles); + + if(null == catagory || catagory.isEmpty()) + return new ResponseEntity(new ResponseDto(HttpStatus.BAD_REQUEST.value(), "invalid catagory!", null), HttpStatus.BAD_REQUEST); + + Object response = metadataService.getDashboardConfiguration(dashboardId, catagory, user.getRoles()); + return new ResponseEntity(new ResponseDto(HttpStatus.OK.value(), "success", response), HttpStatus.OK); + + } catch (Exception e) { + return new ResponseEntity(new ResponseDto(HttpStatus.INTERNAL_SERVER_ERROR.value(), e.getMessage(), null), HttpStatus.INTERNAL_SERVER_ERROR); + + } + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dao/ElasticSearchDao.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dao/ElasticSearchDao.java new file mode 100644 index 000000000..38782121e --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dao/ElasticSearchDao.java @@ -0,0 +1,41 @@ +package com.tarento.analytics.dao; + +import java.util.List; +import java.util.Map; + +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; + +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.AggregateRequestDtoV2; +import com.tarento.analytics.dto.CummulativeDataRequestDto; +import com.tarento.analytics.dto.SearchDto; +import com.tarento.analytics.model.ElasticSearchDictator; + + +public interface ElasticSearchDao { + + public Map getDataByIdentifier(String index, String type, String identifier); + + public Map searchData(String index, String type, Map searchData); + + public Map complexSearch(SearchDto searchDTO, String index, String... type); + + public boolean healthCheck(); + + /*public ElasticSearchDictator createSearchDictator(AggregateRequestDto dto, String tenant) throws Exception ; + + public ElasticSearchDictator createSearchDictator(AggregateRequestDto dto, String indexName, String documentType, String filterDateField) throws Exception ; + */ + public ElasticSearchDictator createSearchDictatorV2(AggregateRequestDtoV2 dto, String indexName, String documentType, String filterDateField) throws Exception ; + + public SearchRequest buildElasticSearchQuery(ElasticSearchDictator dictator); + + public MultiSearchResponse executeMultiSearchRequest(List searchRequestList, String tenant); + + //public ElasticSearchDictator createSearchDictator(String indexName, String documentType, CummulativeDataRequestDto dto, String tenant) throws Exception ; + + MultiSearchResponse executeMultiSearchRequest(List searchRequestList, Boolean primaryOrNot); + +} + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dao/impl/ElasticSearchDaoImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dao/impl/ElasticSearchDaoImpl.java new file mode 100644 index 000000000..336e1657d --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dao/impl/ElasticSearchDaoImpl.java @@ -0,0 +1,1280 @@ +package com.tarento.analytics.dao.impl; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHost; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ExistsQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; +import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.tarento.analytics.constant.ElasticSearchConstants; +import com.tarento.analytics.dao.ElasticSearchDao; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.AggregateRequestDtoV2; +import com.tarento.analytics.dto.CummulativeDataRequestDto; +import com.tarento.analytics.dto.SearchDto; +import com.tarento.analytics.model.ElasticSearchDictator; +import com.tarento.analytics.utils.ElasticProperties; +import com.tarento.analytics.utils.ElasticSearchClient; + +@Component +public class ElasticSearchDaoImpl implements ElasticSearchDao { + + @Autowired + private ElasticSearchClient elasticSearchClient; + + @Autowired + public ObjectMapper mapper; + + private static String elasticHost; + private static int elasticPort; + private static String alternateHost; + private static int alternatePort; + private static final String REST_SCHEME = "http"; + private static final String REST_SCHEME2 = "https"; + private String transactionElasticIndexName; + private static String transactionElasticDocType; + private String ratingElasticIndexName; + private static String ratingElasticDocType; + private static RestHighLevelClient client; + private static RestHighLevelClient alternateClient; + private String indexName; + private String docType; + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getDocType() { + return docType; + } + + public void setDocType(String docType) { + this.docType = docType; + } + + public static final Logger logger = LoggerFactory.getLogger(ElasticSearchDaoImpl.class); + + public ElasticSearchDaoImpl(@Value("${services.esindexer.host.name}") String elasticHost, + @Value("${services.esindexer.host.port}") int elasticPort, + @Value("${services.esindexer.alternate.host.name}") String alternateHost, + @Value("${services.esindexer.alternate.host.port}") int alternatePort) throws MalformedURLException { + this.elasticHost = elasticHost; + this.elasticPort = elasticPort; + this.alternateHost = alternateHost; + this.alternatePort = alternatePort; + this.client = getClientForElastic(); + this.alternateClient = getClientForAlternate(); + } + + @Override + public Map getDataByIdentifier(String index, String type, String identifier) { + RestHighLevelClient client = elasticSearchClient.getClient(); + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil getDataByIdentifier method started at ==" + startTime + " for Type " + type); + GetResponse response = null; + + try { + if (StringUtils.isBlank(index) || StringUtils.isBlank(identifier)) { + logger.error("Invalid request is coming."); + return new HashMap<>(); + } else if (StringUtils.isBlank(type)) { + response = client.get(new GetRequest(index).type(type)); + + } else { + response = client.get(new GetRequest(index, type, identifier)); + } + if (response == null || null == response.getSource()) { + return new HashMap<>(); + } + long stopTime = System.currentTimeMillis(); + long elapsedTime = stopTime - startTime; + logger.info("ElasticSearchUtil getDataByIdentifier method end at ==" + stopTime + " for Type " + type + + " ,Total time elapsed = " + elapsedTime); + return response.getSource(); + } catch (IOException ex) { + + } finally { + try { + client.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + return null; + } + + @Override + public Map searchData(String index, String type, Map searchData) { + RestHighLevelClient client = elasticSearchClient.getClient(); + + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil searchData method started at ==" + startTime + " for Type " + type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + Iterator> itr = searchData.entrySet().iterator(); + while (itr.hasNext()) { + Entry entry = itr.next(); + sourceBuilder.query(QueryBuilders.commonTermsQuery(entry.getKey(), entry.getValue())); + } + + SearchResponse sr = null; + try { + sr = client.search(new SearchRequest(index).types(type).source(sourceBuilder)); + + } catch (IOException ex) { + logger.error("Error while execution in Elasticsearch", ex); + } + + if (sr.getHits() == null || sr.getHits().getTotalHits() == 0) { + return new HashMap<>(); + } + sr.getHits().getAt(0).getSourceAsMap(); + long stopTime = System.currentTimeMillis(); + long elapsedTime = stopTime - startTime; + + logger.info("ElasticSearchUtil searchData method end at ==" + stopTime + " for Type " + type + + " ,Total time elapsed = " + elapsedTime); + try { + client.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + logger.error("Error while closing the client"); + } + return sr.getAggregations().asList().get(0).getMetaData(); + } + + @SuppressWarnings("rawtypes") + @Override + public Map complexSearch(SearchDto searchDTO, String index, String... type) { + RestHighLevelClient client = elasticSearchClient.getClient(); + + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil complexSearch method started at ==" + startTime); + + // Map constraintsMap = getConstraints(searchDTO); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0); + BoolQueryBuilder query = new BoolQueryBuilder(); + + if (!StringUtils.isBlank(searchDTO.getQuery())) { + query.must(QueryBuilders.simpleQueryStringQuery(searchDTO.getQuery()).field("all_fields")); + } + + // apply sorting + if (searchDTO.getSortBy() != null && searchDTO.getSortBy().size() > 0) { + + for (Map.Entry entry : searchDTO.getSortBy().entrySet()) { + SortBuilder sortB = SortBuilders.fieldSort(entry.getKey()).order(getSortOrder(entry.getValue())); + searchSourceBuilder.sort(sortB); + + } + } + + // apply the fields filter + searchSourceBuilder.fetchSource( + searchDTO.getFields() != null ? searchDTO.getFields().stream().toArray(String[]::new) : null, + searchDTO.getExcludedFields() != null ? searchDTO.getExcludedFields().stream().toArray(String[]::new) + : null); + + // setting the offset + if (searchDTO.getOffset() != null) { + searchSourceBuilder.from(searchDTO.getOffset()); + } + + // setting the limit + if (searchDTO.getLimit() != null) { + searchSourceBuilder.size(searchDTO.getLimit()); + } + + // apply additional properties + if (searchDTO.getAdditionalProperties() != null && searchDTO.getAdditionalProperties().size() > 0) { + for (Map.Entry entry : searchDTO.getAdditionalProperties().entrySet()) { + addAdditionalProperties(query, entry); + } + } + + searchSourceBuilder.query(query); + List finalFacetList = new ArrayList(); + + if (null != searchDTO.getFacets() && !searchDTO.getFacets().isEmpty()) { + // addAggregations(searchSourceBuilder, searchDTO.getFacets()); + } + logger.info("calling search builder======" + searchSourceBuilder.toString()); + SearchResponse response = null; + SearchRequest searchReq = new SearchRequest(index).types(type).source(searchSourceBuilder); + List> esSource = new ArrayList<>(); + + try { + response = client.search(searchReq); + long count = 0; + + if (response != null) { + SearchHits hits = response.getHits(); + count = hits.getTotalHits(); + for (SearchHit hit : hits) { + esSource.add(hit.getSourceAsMap()); + } + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return null; + + } + + @SuppressWarnings("unchecked") + private void addAggregations(SearchSourceBuilder searchSourceBuilder, Map aggregations) { + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil addAggregations method started at ==" + startTime); + for (Map.Entry entry : aggregations.entrySet()) { + + String key = entry.getKey(); + Map aggregationInfo = (Map) entry.getValue(); + for (Map.Entry en : aggregations.entrySet()) { + if ("DATE_HISTOGRAM".equalsIgnoreCase(en.getKey())) { + Map aggsVal = (Map) en.getValue(); + DateHistogramInterval dateHistogramInterval = null; + if (aggsVal.get("interval").equals("day")) { + dateHistogramInterval = DateHistogramInterval.DAY; + } else if (aggsVal.get("interval").equals("hour")) { + dateHistogramInterval = DateHistogramInterval.HOUR; + + } else if (aggsVal.get("interval").equals("month")) { + dateHistogramInterval = DateHistogramInterval.MONTH; + + } else if (aggsVal.get("interval").equals("year")) { + dateHistogramInterval = DateHistogramInterval.YEAR; + + } + searchSourceBuilder.aggregation(AggregationBuilders.dateHistogram(key).field(aggsVal.get("field")) + .dateHistogramInterval(dateHistogramInterval)); + + } else if ("TERMS".equalsIgnoreCase(en.getKey())) { + Map aggVal = (Map) en.getValue(); + for (Map.Entry entryS : aggVal.entrySet()) { + + /* + * searchSourceBuilder.aggregation(AggregationBuilders. + * dateHistogram(key).field(aggsVal.get("field")) + * .dateHistogramInterval(Date)); + */ + + } + + } + } + + } + long stopTime = System.currentTimeMillis(); + long elapsedTime = stopTime - startTime; + logger.info("ElasticSearchUtil addAggregations method end at ==" + stopTime + " ,Total time elapsed = " + + elapsedTime); + } + + @SuppressWarnings("unchecked") + private void addAdditionalProperties(BoolQueryBuilder query, Entry entry) { + long startTime = System.currentTimeMillis(); + logger.info("ElasticSearchUtil addAdditionalProperties method started at ==" + startTime); + String key = entry.getKey(); + + if (key.equalsIgnoreCase("FILTERS")) { + + Map filters = (Map) entry.getValue(); + for (Map.Entry en : filters.entrySet()) { + createFilterESOpperation(en, query); + } + } else if (key.equalsIgnoreCase("EXISTS") || key.equalsIgnoreCase("NOT_EXISTS")) { + createESOpperation(entry, query); + } + long stopTime = System.currentTimeMillis(); + long elapsedTime = stopTime - startTime; + logger.info("ElasticSearchUtil addAdditionalProperties method end at ==" + stopTime + " ,Total time elapsed = " + + elapsedTime); + } + + /** Method to create EXISTS and NOT EXIST FILTER QUERY . */ + @SuppressWarnings("unchecked") + private void createESOpperation(Entry entry, BoolQueryBuilder query) { + + String operation = entry.getKey(); + List existsList = (List) entry.getValue(); + + if (operation.equalsIgnoreCase("EXISTS")) { + for (String name : existsList) { + query.must(createExistQuery(name)); + } + } else if (operation.equalsIgnoreCase("NOT_EXISTS")) { + for (String name : existsList) { + query.mustNot(createExistQuery(name)); + } + } + } + + /** Method to create CommonTermQuery , multimatch and Range Query. */ + @SuppressWarnings("unchecked") + private void createFilterESOpperation(Entry entry, BoolQueryBuilder query) { + + String key = entry.getKey(); + Object val = entry.getValue(); + if (val instanceof List) { + if (!((List) val).isEmpty()) { + if (((List) val).get(0) instanceof String) { + ((List) val).replaceAll(String::toLowerCase); + query.must(createTermsQuery(key, (List) val)); + } else { + query.must(createTermsQuery(key, (List) val)); + } + } + } else if (val instanceof Map) { + Map value = (Map) val; + Map rangeOperation = new HashMap<>(); + Map lexicalOperation = new HashMap<>(); + for (Map.Entry it : value.entrySet()) { + String operation = it.getKey(); + if (operation.startsWith(ElasticSearchConstants.LT) + || operation.startsWith(ElasticSearchConstants.GT)) { + rangeOperation.put(operation, it.getValue()); + } else if (operation.startsWith(ElasticSearchConstants.STARTS_WITH) + || operation.startsWith(ElasticSearchConstants.ENDS_WITH)) { + lexicalOperation.put(operation, it.getValue()); + } + } + if (!(rangeOperation.isEmpty())) { + query.must(createRangeQuery(key, rangeOperation)); + } + if (!(lexicalOperation.isEmpty())) { + query.must(createLexicalQuery(key, lexicalOperation)); + } + + } else if (val instanceof String) { + query.must(createTermQuery(key, ((String) val).toLowerCase())); + } else { + query.must(createTermQuery(key, val)); + } + } + + private static TermQueryBuilder createTermQuery(String name, Object text) { + + return QueryBuilders.termQuery(name, text); + + } + + @SuppressWarnings("unchecked") + private static TermsQueryBuilder createTermsQuery(String key, List values) { + + return QueryBuilders.termsQuery(key, (values).stream().toArray(Object[]::new)); + + } + + private static Map getConstraints(SearchDto searchDTO) { + if (null != searchDTO.getSoftConstraints() && !searchDTO.getSoftConstraints().isEmpty()) { + return searchDTO.getSoftConstraints().entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().floatValue())); + } + return Collections.emptyMap(); + } + + @Override + public boolean healthCheck() { + // TODO Auto-generated method stub + return false; + } + + /// New Methods for Elastic Search Query Builders + + private RangeQueryBuilder createRangeQuery(String name, Map rangeOperation) { + + RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(name); + for (Map.Entry it : rangeOperation.entrySet()) { + if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.LTE)) { + rangeQueryBuilder.lte(it.getValue()); + } else if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.LT)) { + rangeQueryBuilder.lt(it.getValue()); + } else if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.GTE)) { + rangeQueryBuilder.gte(it.getValue()); + } else if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.GT)) { + rangeQueryBuilder.gt(it.getValue()); + } + } + + return rangeQueryBuilder; + } + + private SortOrder getSortOrder(String value) { + return value.equalsIgnoreCase(ElasticSearchConstants.ASC_ORDER) ? SortOrder.ASC : SortOrder.DESC; + } + + private QueryBuilder createLexicalQuery(String key, Map rangeOperation) { + QueryBuilder queryBuilder = null; + for (Map.Entry it : rangeOperation.entrySet()) { + if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.STARTS_WITH)) { + + queryBuilder = QueryBuilders.prefixQuery(key, (String) it.getValue()); + } else if (it.getKey().equalsIgnoreCase(ElasticSearchConstants.ENDS_WITH)) { + String endsWithRegex = "~" + it.getValue(); + queryBuilder = QueryBuilders.regexpQuery(key, endsWithRegex); + } + } + return queryBuilder; + } + + private ExistsQueryBuilder createExistQuery(String name) { + return QueryBuilders.existsQuery(name); + } +/* + + @SuppressWarnings("unchecked") + @Override + public ElasticSearchDictator createSearchDictator(AggregateRequestDto dto, String tenant) throws Exception { + ElasticSearchDictator dictator = new ElasticSearchDictator(); + if (dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_WRT_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_DAY_WISE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_REASON) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_GENDER) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_AGEGROUP) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_TOP_PERFORMING) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_LEAST_PERFORMING)) { + dictator.setIndexName(tenant + ratingElasticIndexName); + dictator.setDocumentType(ratingElasticDocType); + } else { + dictator.setIndexName(tenant + transactionElasticIndexName); + dictator.setDocumentType(transactionElasticDocType); + } + if (StringUtils.isNotBlank(dto.getServiceApi())) { + dictator.setVisualisationName(dto.getServiceApi()); + } + Map>> queryMap = new HashMap<>(); + if (dto.getCustomData() != null) { + for (Map.Entry entry : dto.getCustomData().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + List valueList = new ArrayList<>(); + + if (entry.getValue() instanceof ArrayList) { + + List valueArray = (ArrayList) entry.getValue(); + + for (Object value : valueArray) { + valueList.add(value); + } + } else { + valueList.add(entry.getValue()); + } + if (!valueList.isEmpty()) { + String entryKey = entry.getKey(); + if (dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_WRT_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_DAY_WISE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_REASON) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_GENDER) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_AGEGROUP) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_TOP_PERFORMING) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_LEAST_PERFORMING)) { + + entryKey = "storeId"; + } + if (queryMap.containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + Map> queryInnerMap = queryMap + .get(ElasticProperties.Query.MATCH_CONDITION); + queryInnerMap.put(entryKey, valueList); + } else { + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(entryKey, valueList); + queryMap.put(ElasticProperties.Query.MATCH_CONDITION, queryInnerMap); + } + } + } + } + } + + if (dto.getDates() != null) { + if (StringUtils.isNotBlank(dto.getDates().getStartDate()) + && StringUtils.isNotBlank(dto.getDates().getEndDate())) { + List valueList = new ArrayList<>(); + + valueList.add(dto.getDates().getStartDate()); + valueList.add(dto.getDates().getEndDate()); + Map> queryInnerMap = new HashMap<>(); + if (dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_WRT_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_REASON) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_GENDER) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_AGEGROUP) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_STORE_TIME) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_TOP_PERFORMING) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_LEAST_PERFORMING) + || dto.getServiceApi().equals(ServiceApiConstants.CSAT_DAY_WISE)) { + queryInnerMap.put(ElasticProperties.Query.FEEDBACK_DATE_TIME, valueList); + } else { + queryInnerMap.put(ElasticProperties.Query.TRANSACTION_DATE_FIELD, valueList); + } + + queryMap.put(ElasticProperties.Query.RANGE_CONDITION, queryInnerMap); + } + } + dictator.setQueryMap(queryMap); + + */ +/* + * Map>> aggregationMap = new + * HashMap<>(); Map> innerMap = new + * HashMap<>(); Map deepInnerMap = new HashMap<>(); + *//* + + + return dictator; + + } +*/ + + @SuppressWarnings("unchecked") + @Override + public SearchRequest buildElasticSearchQuery(ElasticSearchDictator dictator) { + SearchSourceBuilder searchBuilder = buildSearchSourceBuilder(); + BoolQueryBuilder boolQuery = buildBoolQuery(); + String localDateStartRange = null; + String localDateEndRange = null; + if (dictator.getQueryMap().containsKey(ElasticProperties.Query.RANGE_CONDITION)) { + + String searchParamRange = null; + Map> innerMap = dictator.getQueryMap().get(ElasticProperties.Query.RANGE_CONDITION); + for (Entry> en : innerMap.entrySet()) { + searchParamRange = en.getKey(); + localDateStartRange = (String) en.getValue().get(0); + localDateEndRange = (String) en.getValue().get(1); + } + + addMustOnBoolQuery(boolQuery, buildRangeQuery(searchParamRange, localDateStartRange, localDateEndRange)); + + } + + if (dictator.getQueryMap().containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + String searchTermField = null; + Map> innerMap = dictator.getQueryMap().get(ElasticProperties.Query.MATCH_CONDITION); + for (Entry> en : innerMap.entrySet()) { + searchTermField = en.getKey(); + addFilterTermsOnBoolQuery(boolQuery, buildTermsQuery(searchTermField, en.getValue())); + } + } + + addQueryToSearchBuilder(searchBuilder, boolQuery); + + DateHistogramAggregationBuilder dateAggBuilder = null; + AvgAggregationBuilder avgAggBuilder = null; + SumAggregationBuilder sumAggBuilder = null; + TermsAggregationBuilder termsAggBuilder = null; + NestedAggregationBuilder nestedAggBuilder = null; + ValueCountAggregationBuilder valueCountAggBuilder = null; + if(dictator.getQueryAggregationMap()==null) { + return new SearchRequest(dictator.getIndexName()).types(dictator.getDocumentType()).source(searchBuilder); + } + + for (Map.Entry itr : dictator.getQueryAggregationMap().entrySet()) { + + String aggregationType = itr.getKey(); + Object value = itr.getValue(); + if (ElasticProperties.Query.NESTED.equals(aggregationType.toUpperCase())) { + Map nestedMap = null; + if (value instanceof HashMap) { + nestedMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + nestedMap = (LinkedHashMap) value; + } + String aggregationName = aggregationType + "of" + + nestedMap.get(ElasticProperties.Query.PATH.toLowerCase()); + aggregationName = String.valueOf(nestedMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + nestedAggBuilder = buildNestedAggregationBuilder(aggregationName, nestedMap); + + } else if (ElasticProperties.Query.SUM.equals(aggregationType.toUpperCase())) { + Map sumMap = null; + if (value instanceof HashMap) { + sumMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + sumMap = (LinkedHashMap) value; + } + String aggregationName = aggregationType + "of" + + sumMap.get(ElasticProperties.Query.FIELD.toLowerCase()); + aggregationName = String.valueOf(sumMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + sumAggBuilder = buildSumAggregation(aggregationName, + String.valueOf(sumMap.get(ElasticProperties.Query.FIELD.toLowerCase()))); + } else if (ElasticProperties.Query.TERM.equals(aggregationType.toUpperCase())) { + Map termMap = null; + if (value instanceof HashMap) { + termMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + termMap = (LinkedHashMap) value; + + } + String aggregationName = aggregationType + "of" + + termMap.get(ElasticProperties.Query.FIELD.toLowerCase()); + aggregationName = String.valueOf(termMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + termsAggBuilder = buildTermAggregation(aggregationName, termMap); + + } else if (ElasticProperties.Query.DATE_HISTOGRAM.equals(aggregationType.toUpperCase())) { + Map histogramMap = null; + + if (value instanceof HashMap) { + histogramMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + histogramMap = (LinkedHashMap) value; + + } + String aggregationName = aggregationType + "of" + + histogramMap.get(ElasticProperties.Query.FIELD.toLowerCase()); + aggregationName = String.valueOf(histogramMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + dateAggBuilder = buildDateHistogramAggregation(aggregationName, + histogramMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(), + histogramMap.get(ElasticProperties.Query.INTERVAL.toLowerCase()).toString().toUpperCase(), + Long.parseLong(localDateStartRange), Long.parseLong(localDateEndRange)); + + } else if (ElasticProperties.Query.COUNT.equals(aggregationType.toUpperCase())) { + Map countAggMap = null; + + if (value instanceof HashMap) { + countAggMap = (HashMap) value; + } else if (value instanceof LinkedHashMap) { + countAggMap = (LinkedHashMap) value; + + } + String countField = countAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + String countAggName = aggregationType + "of" + countField; + countAggName = String.valueOf(countAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + valueCountAggBuilder = buildCountsAggregationQuery(countAggName, countField); + } else if (ElasticProperties.Query.AGGREGATION_CONDITION.equals(aggregationType.toUpperCase())) { + + Map firstLevelAggMap = (LinkedHashMap) value; + for (Map.Entry firstLevelItrEntry : firstLevelAggMap.entrySet()) { + + String firstLevelEntryKey = firstLevelItrEntry.getKey(); + Object firstLevelEntryValue = firstLevelItrEntry.getValue(); + if (ElasticProperties.Query.SUM.equals(firstLevelEntryKey.toUpperCase())) { + Map sumAggMap = (LinkedHashMap) firstLevelEntryValue; + String sumField = sumAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + String sumAggName = firstLevelEntryKey + "of" + sumField; + sumAggName = String.valueOf(sumAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + if (termsAggBuilder != null) { + termsAggBuilder = buildSubSumAggreationForTerms(termsAggBuilder, sumAggName, sumField); + } else if (dateAggBuilder != null) { + dateAggBuilder = buildSubSumAggreationForDateHistogram(dateAggBuilder, sumAggName, + sumField); + } + } else if (ElasticProperties.Query.AVG.equals(firstLevelEntryKey.toUpperCase())) { + Map avgAggMap = (LinkedHashMap) firstLevelEntryValue; + String avgField = avgAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + String avgAggName = firstLevelEntryKey + "of" + avgField; + avgAggName = String.valueOf(avgAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + if (termsAggBuilder != null) { + termsAggBuilder = buildSubAvgAggreationForTerms(termsAggBuilder, avgAggName, avgField); + } else if (dateAggBuilder != null) { + dateAggBuilder = buildSubSumAggreationForDateHistogram(dateAggBuilder, avgAggName, + avgField); + } + } else if (ElasticProperties.Query.COUNT.equals(firstLevelEntryKey.toUpperCase())) { + Map countAggMap = (LinkedHashMap) firstLevelEntryValue; + String countField = countAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + String countAggName = firstLevelEntryKey + "of" + countField; + countAggName = String.valueOf(countAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + if (termsAggBuilder != null) { + termsAggBuilder = buildSubCountAggregationForTerms(termsAggBuilder, countAggName, + countField); + } else if (dateAggBuilder != null) { + dateAggBuilder = buildSubCountAggregationForDateHistogram(dateAggBuilder, countAggName, + countField); + } + } else if (ElasticProperties.Query.TERM.equals(firstLevelEntryKey.toUpperCase())) { + Map subTermAggMap = (LinkedHashMap) firstLevelEntryValue; + String subTermAggName = firstLevelEntryKey + "of" + + subTermAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString(); + subTermAggName = String.valueOf(subTermAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + if (nestedAggBuilder != null) { + buildSubTermsAggregationForNested(nestedAggBuilder, subTermAggName, + subTermAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString()); + } else if (dateAggBuilder != null) { + buildSubTermsAggregationForHistogram(dateAggBuilder, subTermAggName, + subTermAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString()); + } else if (termsAggBuilder != null) { + buildSubTermsAggregationForTerms(termsAggBuilder, subTermAggName, + subTermAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()).toString()); + } + + } else if (ElasticProperties.Query.AGGREGATION_CONDITION.equals(firstLevelEntryKey.toUpperCase())) { + + Map secondLevelAggMap = (LinkedHashMap) firstLevelEntryValue; + for (Map.Entry secondLevelItrEntry : secondLevelAggMap.entrySet()) { + String secondLevelItrEntryKey = secondLevelItrEntry.getKey(); + Object secondLevelItrEntryValue = secondLevelItrEntry.getValue(); + if (ElasticProperties.Query.SUM.equals(secondLevelItrEntryKey.toUpperCase())) { + Map subSumAggMap = (LinkedHashMap) secondLevelItrEntryValue; + + String subSumField = subSumAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()) + .toString(); + String subSumAggName = secondLevelItrEntryKey + "of" + subSumField; + subSumAggName = String.valueOf(subSumAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + + if (nestedAggBuilder != null) { + List aggBuilders = nestedAggBuilder.getSubAggregations(); + for (AggregationBuilder aggregationBuilder : aggBuilders) { + + if (aggregationBuilder instanceof TermsAggregationBuilder) { + TermsAggregationBuilder subTermAggBuilder = (TermsAggregationBuilder) aggregationBuilder; + subTermAggBuilder = buildSubSumAggreationForTerms(subTermAggBuilder, + subSumAggName, subSumField); + + } + } + } + } else if (ElasticProperties.Query.COUNT.equals(secondLevelItrEntryKey.toUpperCase())) { + Map subCountAggMap = (LinkedHashMap) secondLevelItrEntryValue; + + String subCountField = subCountAggMap.get(ElasticProperties.Query.FIELD.toLowerCase()) + .toString(); + String subCountAggName = secondLevelItrEntryKey + "of" + subCountField; + subCountAggName = String.valueOf(subCountAggMap.get(ElasticProperties.Query.LABEL.toLowerCase())); + + if (dateAggBuilder != null) { + List aggBuilders = dateAggBuilder.getSubAggregations(); + for (AggregationBuilder aggregationBuilder : aggBuilders) { + + if (aggregationBuilder instanceof TermsAggregationBuilder) { + TermsAggregationBuilder subTermAggBuilder = (TermsAggregationBuilder) aggregationBuilder; + subTermAggBuilder = buildSubCountAggregationForTerms(subTermAggBuilder, + subCountAggName, subCountField); + } + } + } + } + } + } + } + } + } + + if (dateAggBuilder != null) { + addAggregationToSearchBuilder(searchBuilder, dateAggBuilder); + } + if (sumAggBuilder != null) { + addSumAggregationToSearchBuilder(searchBuilder, sumAggBuilder); + } + if (avgAggBuilder != null) { + addAvgAggregationToSearchBuilder(searchBuilder, avgAggBuilder); + } + if (valueCountAggBuilder != null) { + addValueCountAggregationToSearchBuilder(searchBuilder, valueCountAggBuilder); + } + if (termsAggBuilder != null) { + addTermsAggregationToSearchBuilder(searchBuilder, termsAggBuilder); + } + if (nestedAggBuilder != null) { + addNestedAggregationToSearchBuilder(searchBuilder, nestedAggBuilder); + } + return new SearchRequest(dictator.getIndexName()).types(dictator.getDocumentType()).source(searchBuilder); + } + + private NestedAggregationBuilder buildSubTermsAggregationForNested(NestedAggregationBuilder nestedAggBuilder, + String subAggregationName, String fieldName) { + return nestedAggBuilder.subAggregation(AggregationBuilders.terms(subAggregationName).field(fieldName)); + + } + + private void addNestedAggregationToSearchBuilder(SearchSourceBuilder searchBuilder, + NestedAggregationBuilder nestedAggBuilder) { + searchBuilder.aggregation(nestedAggBuilder); + } + + private RangeQueryBuilder buildRangeQuery(String fieldName, Object startRange, Object endRange) { + return QueryBuilders.rangeQuery(fieldName).gte(startRange).lte(endRange); + } + + private TermQueryBuilder buildTermQuery(String fieldName, Object fieldValue) { + return QueryBuilders.termQuery(fieldName, fieldValue); + } + + private TermsQueryBuilder buildTermsQuery(String fieldName, List fieldValue) { + return QueryBuilders.termsQuery(fieldName, fieldValue); + } + + private MatchQueryBuilder buildMatchQuery(String fieldName, Object fieldValue) { + return QueryBuilders.matchQuery(fieldName, fieldValue); + } + + private ValueCountAggregationBuilder buildCountsAggregationQuery(String aggregationName, String fieldName) { + return AggregationBuilders.count(aggregationName).field(fieldName); + } + + private SumAggregationBuilder buildSumAggregation(String aggregationName, String fieldName) { + return AggregationBuilders.sum(aggregationName).field(fieldName); + } + + @SuppressWarnings("unchecked") + private TermsAggregationBuilder buildTermAggregation(String aggregationName, Map paramMap) { + TermsAggregationBuilder aggBuilder = AggregationBuilders.terms(aggregationName); + for (Map.Entry param : paramMap.entrySet()) { + if (param.getKey().toUpperCase().equals(ElasticProperties.Query.FIELD)) { + aggBuilder = aggBuilder.field((String) param.getValue()); + } else if (param.getKey().toUpperCase().equals(ElasticProperties.Query.SIZE)) { + aggBuilder = aggBuilder.size((Integer) param.getValue()); + } else if (param.getKey().toUpperCase().equals(ElasticProperties.Query.ORDER)) { + Map keyMap = (HashMap) param.getValue(); + BucketOrder order = null; + for (Map.Entry valueMap : keyMap.entrySet()) { + Map orderMap = (HashMap) valueMap.getValue(); + + String key = valueMap.getKey() + "of" + orderMap.get(ElasticProperties.Query.FIELD.toLowerCase()); + + order = BucketOrder.aggregation(key, + orderMap.get("orderBy").toUpperCase().equals(ElasticProperties.Query.ASC) ? true : false); + } + aggBuilder = aggBuilder.order(order); + } + } + return aggBuilder; + + } + + private AvgAggregationBuilder buildAvgAggregation(String aggregationName, String fieldName) { + return AggregationBuilders.avg(aggregationName).field(fieldName); + } + + private NestedAggregationBuilder buildNestedAggregationBuilder(String aggregationName, + Map paramMap) { + String pathName = paramMap.get(ElasticProperties.Query.PATH.toLowerCase()).toString(); + return AggregationBuilders.nested(aggregationName, pathName); + } + + private AggregationBuilder buildSubTermForNestedAggregation(NestedAggregationBuilder nestedBuilder, + String aggregationName, Map paramMap) { + return nestedBuilder.subAggregation(buildTermAggregation(aggregationName, paramMap)); + + } + + private DateHistogramAggregationBuilder buildDateHistogramAggregation(String aggregationName, String fieldName, + String interval, Long boundMin, Long boundMax) { + DateHistogramInterval dateHistogramInterval = null; + if (ElasticProperties.Query.HOUR.equals(interval)) { + dateHistogramInterval = DateHistogramInterval.HOUR; + } else if (ElasticProperties.Query.DAY.equals(interval)) { + dateHistogramInterval = DateHistogramInterval.DAY; + } else if (ElasticProperties.Query.MINUTE.equals(interval)) { + dateHistogramInterval = DateHistogramInterval.MINUTE; + } else if (ElasticProperties.Query.MONTH.equals(interval)) { + dateHistogramInterval = DateHistogramInterval.MONTH; + + } + return AggregationBuilders.dateHistogram(aggregationName).field(fieldName) + .dateHistogramInterval(dateHistogramInterval).format("epoch_millis") + .extendedBounds(new ExtendedBounds(boundMin, boundMax)); + } + + private DateHistogramAggregationBuilder buildSubSumAggreationForDateHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.sum(aggregationName).field(fieldName)); + } + + private DateHistogramAggregationBuilder buildSubCummulativeSumAggreationForDateHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + PipelineAggregationBuilder pipeLineAggregation = new CumulativeSumPipelineAggregationBuilder(aggregationName, + fieldName); + return builder.subAggregation(pipeLineAggregation); + } + + private DateHistogramAggregationBuilder buildSubAvgAggreationForDateHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.avg(aggregationName).field(fieldName)); + } + + private DateHistogramAggregationBuilder buildSubCountAggregationForDateHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.count(aggregationName).field(fieldName)); + } + + private TermsAggregationBuilder buildSubCountAggregationForTerms(TermsAggregationBuilder builder, + String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.count(aggregationName).field(fieldName)); + } + + private TermsAggregationBuilder buildSubTermsAggregationForTerms(TermsAggregationBuilder builder, + String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.terms(aggregationName).field(fieldName)); + } + + private DateHistogramAggregationBuilder buildSubTermsAggregationForHistogram( + DateHistogramAggregationBuilder builder, String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.terms(aggregationName).field(fieldName)); + } + + private TermsAggregationBuilder buildSubSumAggreationForTerms(TermsAggregationBuilder builder, + String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.sum(aggregationName).field(fieldName)); + } + + private TermsAggregationBuilder buildSubAvgAggreationForTerms(TermsAggregationBuilder builder, + String aggregationName, String fieldName) { + return builder.subAggregation(AggregationBuilders.avg(aggregationName).field(fieldName)); + } + + private SumAggregationBuilder buildSubSumAggregation(SumAggregationBuilder builder, String aggregationName, + String fieldName) { + return builder.subAggregation(AggregationBuilders.sum(aggregationName).field(fieldName)); + } + + private AvgAggregationBuilder buildSubAvgAggregation(AvgAggregationBuilder builder, String aggregationName, + String fieldName) { + return builder.subAggregation(AggregationBuilders.avg(aggregationName).field(fieldName)); + } + + private BoolQueryBuilder buildBoolQuery() { + return QueryBuilders.boolQuery(); + } + + private BoolQueryBuilder addMustOnBoolQuery(BoolQueryBuilder builder, RangeQueryBuilder rangeBuilder) { + return builder.must(rangeBuilder); + } + + private BoolQueryBuilder addFilterOnBoolQuery(BoolQueryBuilder builder, TermQueryBuilder termBuilder) { + return builder.filter(termBuilder); + } + + private BoolQueryBuilder addFilterTermsOnBoolQuery(BoolQueryBuilder builder, TermsQueryBuilder termBuilder) { + return builder.filter(termBuilder); + } + + private BoolQueryBuilder addMatchOnBoolQuery(BoolQueryBuilder builder, MatchQueryBuilder matchBuilder) { + return builder.filter(matchBuilder); + } + + private SearchSourceBuilder buildSearchSourceBuilder() { + return new SearchSourceBuilder().size(1); + } + + private SearchSourceBuilder addQueryToSearchBuilder(SearchSourceBuilder builder, BoolQueryBuilder queryBuilder) { + return builder.query(queryBuilder); + } + + private SearchSourceBuilder addAggregationToSearchBuilder(SearchSourceBuilder builder, + DateHistogramAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + + private SearchSourceBuilder addSumAggregationToSearchBuilder(SearchSourceBuilder builder, + SumAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + + private SearchSourceBuilder addAvgAggregationToSearchBuilder(SearchSourceBuilder builder, + AvgAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + + private SearchSourceBuilder addValueCountAggregationToSearchBuilder(SearchSourceBuilder builder, + ValueCountAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + private SearchSourceBuilder addTermsAggregationToSearchBuilder(SearchSourceBuilder builder, + TermsAggregationBuilder aggBuilder) { + return builder.aggregation(aggBuilder); + } + + @Override + public MultiSearchResponse executeMultiSearchRequest(List searchRequestList, String tenant) { + MultiSearchRequest multiRequest = new MultiSearchRequest(); + MultiSearchResponse response = null; + + for (SearchRequest request : searchRequestList) { + logger.info("ES Query is : " + request.source()); + multiRequest.add(request); + } + + try { + response = client.multiSearch(multiRequest); + } catch (IOException e) { + logger.error("Encountered an error while connecting : " + e); + logger.error("Error Message to report : " + e.getMessage()); + } + + return response; + } + + private RestHighLevelClient getClientForElastic() { + return new RestHighLevelClient(RestClient.builder(new HttpHost(elasticHost, elasticPort, REST_SCHEME))); + } + + private RestHighLevelClient getClientForAlternate() throws MalformedURLException { + URL url = new URL ("https://egov-micro-dev.egovernments.org/elasticsearch/"); + return new RestHighLevelClient(RestClient.builder(new HttpHost(url.getHost(), url.getPort(), REST_SCHEME2))); + } + +/* @SuppressWarnings("unchecked") + @Override + public ElasticSearchDictator createSearchDictator(String indexName, String documentType, + CummulativeDataRequestDto dto, String dateField) throws Exception { + ElasticSearchDictator dictator = new ElasticSearchDictator(); + + dictator.setIndexName(indexName); + dictator.setDocumentType(documentType); + + Map>> queryMap = new HashMap<>(); + if (dto.getCustomData() != null) { + for (Map.Entry entry : dto.getCustomData().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + List valueList = new ArrayList<>(); + + if (entry.getValue() instanceof ArrayList) { + + List valueArray = (ArrayList) entry.getValue(); + + for (Object value : valueArray) { + valueList.add(value); + } + } else { + valueList.add(entry.getValue()); + } + if (!valueList.isEmpty()) { + String entryKey = entry.getKey(); + if (queryMap.containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + Map> queryInnerMap = queryMap + .get(ElasticProperties.Query.MATCH_CONDITION); + queryInnerMap.put(entryKey, valueList); + } else { + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(entryKey, valueList); + queryMap.put(ElasticProperties.Query.MATCH_CONDITION, queryInnerMap); + } + } + } + } + } + + if (dto.getDates() != null) { + if (StringUtils.isNotBlank(dto.getDates().getStartDate()) + && StringUtils.isNotBlank(dto.getDates().getEndDate())) { + List valueList = new ArrayList<>(); + + valueList.add(dto.getDates().getStartDate()); + valueList.add(dto.getDates().getEndDate()); + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(dateField, valueList); + queryMap.put(ElasticProperties.Query.RANGE_CONDITION, queryInnerMap); + } + } + dictator.setQueryMap(queryMap); + + *//* + * Map>> aggregationMap = new + * HashMap<>(); Map> innerMap = new + * HashMap<>(); Map deepInnerMap = new HashMap<>(); + *//* + + return dictator; + } + + @Override + public ElasticSearchDictator createSearchDictator(AggregateRequestDto dto, String indexName, String documentType, + String filterDateField) throws Exception { + ElasticSearchDictator dictator = new ElasticSearchDictator(); + + dictator.setIndexName(indexName); + dictator.setDocumentType(documentType); + if (StringUtils.isNotBlank(dto.getServiceApi())) { + dictator.setVisualisationName(dto.getServiceApi()); + } + + Map>> queryMap = new HashMap<>(); + if (dto.getCustomData() != null) { + for (Map.Entry entry : dto.getCustomData().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + List valueList = new ArrayList<>(); + + if (entry.getValue() instanceof ArrayList) { + + List valueArray = (ArrayList) entry.getValue(); + + for (Object value : valueArray) { + valueList.add(value); + } + } else { + valueList.add(entry.getValue()); + } + if (!valueList.isEmpty()) { + String entryKey = entry.getKey(); + if (queryMap.containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + Map> queryInnerMap = queryMap + .get(ElasticProperties.Query.MATCH_CONDITION); + queryInnerMap.put(entryKey, valueList); + } else { + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(entryKey, valueList); + queryMap.put(ElasticProperties.Query.MATCH_CONDITION, queryInnerMap); + } + } + } + } + } + + if (dto.getDates() != null) { + if (StringUtils.isNotBlank(dto.getDates().getStartDate()) + && StringUtils.isNotBlank(dto.getDates().getEndDate())) { + List valueList = new ArrayList<>(); + + valueList.add(dto.getDates().getStartDate()); + valueList.add(dto.getDates().getEndDate()); + Map> queryInnerMap = new HashMap<>(); + + queryInnerMap.put(filterDateField, valueList); + + queryMap.put(ElasticProperties.Query.RANGE_CONDITION, queryInnerMap); + } + } + dictator.setQueryMap(queryMap); + return dictator; + }*/ + + @Override + public ElasticSearchDictator createSearchDictatorV2(AggregateRequestDtoV2 dto, String indexName, String documentType, + String filterDateField) throws Exception { + ElasticSearchDictator dictator = new ElasticSearchDictator(); + + dictator.setIndexName(indexName); + dictator.setDocumentType(documentType); + + Map>> queryMap = new HashMap<>(); + if (dto.getEsFilters() != null && !dto.getEsFilters().isEmpty()) { + for (Map.Entry entry : dto.getEsFilters().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + List valueList = new ArrayList<>(); + + if (entry.getValue() instanceof ArrayList) { + + List valueArray = (ArrayList) entry.getValue(); + + for (Object value : valueArray) { + valueList.add(value); + } + } else { + valueList.add(entry.getValue()); + } + if (!valueList.isEmpty()) { + String entryKey = entry.getKey(); + if (queryMap.containsKey(ElasticProperties.Query.MATCH_CONDITION)) { + Map> queryInnerMap = queryMap + .get(ElasticProperties.Query.MATCH_CONDITION); + queryInnerMap.put(entryKey, valueList); + } else { + Map> queryInnerMap = new HashMap<>(); + queryInnerMap.put(entryKey, valueList); + queryMap.put(ElasticProperties.Query.MATCH_CONDITION, queryInnerMap); + } + } + } + } + } + + Map queryAggregationMap = new HashMap<>(); + if (dto.getAggregationFactors() != null && !dto.getAggregationFactors().isEmpty()) { + for (Map.Entry entry : dto.getAggregationFactors().entrySet()) { + if (StringUtils.isNotBlank(entry.getKey()) && entry.getValue() != null) { + Map innerAggregationMap = new HashMap<>(); + innerAggregationMap.put("field", entry.getValue()); + queryAggregationMap.put(entry.getKey(), innerAggregationMap); + } + } + } + dictator.setQueryAggregationMap(queryAggregationMap); + if (dto.getRequestDate() != null) { + if (StringUtils.isNotBlank(dto.getRequestDate().getStartDate()) + && StringUtils.isNotBlank(dto.getRequestDate().getEndDate())) { + List valueList = new ArrayList<>(); + + valueList.add(dto.getRequestDate().getStartDate()); + valueList.add(dto.getRequestDate().getEndDate()); + Map> queryInnerMap = new HashMap<>(); + + queryInnerMap.put(filterDateField, valueList); + + queryMap.put(ElasticProperties.Query.RANGE_CONDITION, queryInnerMap); + } + } + dictator.setQueryMap(queryMap); + return dictator; + } + + @Override + public MultiSearchResponse executeMultiSearchRequest(List searchRequestList, Boolean primaryOrNot) { + MultiSearchRequest multiRequest = new MultiSearchRequest(); + MultiSearchResponse response = null; + for (SearchRequest request : searchRequestList) { + logger.info("ES Query is : " + request.source()); + multiRequest.add(request); + } + + try { + if(primaryOrNot) { + response = client.multiSearch(multiRequest); + } else { + logger.info("Alternate Client URL : " + alternateClient.toString()); + response = alternateClient.multiSearch(multiRequest); + } + + } catch (IOException e) { + logger.error("Encountered an error while connecting : " + e); + logger.error("Error Message to report : " + e.getMessage()); + } + return response; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateDto.java new file mode 100644 index 000000000..427dddbe0 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateDto.java @@ -0,0 +1,85 @@ +package com.tarento.analytics.dto; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.tarento.analytics.enums.ChartType; + +/** + * @author Darshan Nagesh + * + */ +public class AggregateDto { + + private ChartType chartType; + + private String chartFormat; + + private String drillDownChartId; + + public String getDrillDownChartId() { + return drillDownChartId; + } + + public void setDrillDownChartId(String drillDownChartId) { + this.drillDownChartId = drillDownChartId; + } + + private Map customData; + + private RequestDate dates; + + private Object filter; + + private List data = new ArrayList<>(); + + public List getData() { + return data; + } + + public void setData(List data) { + this.data = data; + } + + public ChartType getChartType() { + return chartType; + } + + public void setChartType(ChartType chartType) { + this.chartType = chartType; + } + + public String getChartFormat() { + return chartFormat; + } + + public void setChartFormat(String chartFormat) { + this.chartFormat = chartFormat; + } + + + public Map getCustomData() { + return customData; + } + + public void setCustomData(Map customData) { + this.customData = customData; + } + + public RequestDate getDates() { + return dates; + } + + public void setDates(RequestDate dates) { + this.dates = dates; + } + + public Object getFilter() { + return filter; + } + + public void setFilter(Object filter) { + this.filter = filter; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDto.java new file mode 100644 index 000000000..891bfea66 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDto.java @@ -0,0 +1,77 @@ +package com.tarento.analytics.dto; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tarento.analytics.enums.ChartType; + +public class AggregateRequestDto { + + @JsonProperty("chartType") + private ChartType chartType; + + @JsonProperty("chartFormat") + private String chartFormat; + + @JsonProperty("serviceApi") + private String serviceApi; + + @JsonProperty("customData") + private Map customData; + + @JsonProperty("dates") + private RequestDate dates; + + @JsonProperty("interval") + private String interval; + + public ChartType getChartType() { + return chartType; + } + + public void setChartType(ChartType chartType) { + this.chartType = chartType; + } + + public String getChartFormat() { + return chartFormat; + } + + public void setChartFormat(String chartFormat) { + this.chartFormat = chartFormat; + } + + public String getServiceApi() { + return serviceApi; + } + + public void setServiceApi(String serviceApi) { + this.serviceApi = serviceApi; + } + + public Map getCustomData() { + return customData; + } + + public void setCustomData(Map customData) { + this.customData = customData; + } + + public RequestDate getDates() { + return dates; + } + + public void setDates(RequestDate dates) { + this.dates = dates; + } + + public String getInterval() { + return interval; + } + + public void setInterval(String interval) { + this.interval = interval; + } +} + + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDtoV2.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDtoV2.java new file mode 100644 index 000000000..20808f6bf --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/AggregateRequestDtoV2.java @@ -0,0 +1,74 @@ +package com.tarento.analytics.dto; + +import java.util.Map; + +public class AggregateRequestDtoV2 { + + private String visualizationType; + private String visualizationCode; + private String moduleLevel; + private String queryType; + private Map filters; + private Map esFilters; + private Map aggregationFactors; + private RequestDate requestDate; + private String interval; + + public String getModuleLevel() { + return moduleLevel; + } + public void setModuleLevel(String moduleLevel) { + this.moduleLevel = moduleLevel; + } + public Map getEsFilters() { + return esFilters; + } + public void setEsFilters(Map esFilters) { + this.esFilters = esFilters; + } + public String getVisualizationCode() { + return visualizationCode; + } + public void setVisualizationCode(String visualizationCode) { + this.visualizationCode = visualizationCode; + } + public String getVisualizationType() { + return visualizationType; + } + public void setVisualizationType(String visualizationType) { + this.visualizationType = visualizationType; + } + public String getQueryType() { + return queryType; + } + public void setQueryType(String queryType) { + this.queryType = queryType; + } + public Map getFilters() { + return filters; + } + public void setFilters(Map filters) { + this.filters = filters; + } + public Map getAggregationFactors() { + return aggregationFactors; + } + public void setAggregationFactors(Map aggregationFactors) { + this.aggregationFactors = aggregationFactors; + } + public RequestDate getRequestDate() { + return requestDate; + } + public void setRequestDate(RequestDate requestDate) { + this.requestDate = requestDate; + } + public String getInterval() { + return interval; + } + public void setInterval(String interval) { + this.interval = interval; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/CummulativeDataRequestDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/CummulativeDataRequestDto.java new file mode 100644 index 000000000..3ba4bf8e7 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/CummulativeDataRequestDto.java @@ -0,0 +1,32 @@ +package com.tarento.analytics.dto; + +import java.util.List; +import java.util.Map; + +public class CummulativeDataRequestDto { + private Map customData; + private RequestDate dates; + private String dashCode; + + + + public String getDashCode() { + return dashCode; + } + public void setDashCode(String dashCode) { + this.dashCode = dashCode; + } + public Map getCustomData() { + return customData; + } + public void setCustomData(Map customData) { + this.customData = customData; + } + public RequestDate getDates() { + return dates; + } + public void setDates(RequestDate dates) { + this.dates = dates; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardDto.java new file mode 100644 index 000000000..1ff82a8da --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardDto.java @@ -0,0 +1,44 @@ +package com.tarento.analytics.dto; + +public class DashboardDto { + + private Long id; + private String name; + private String code; + private String description; + private String placement; + + public String getPlacement() { + return placement; + } + public void setPlacement(String placement) { + this.placement = placement; + } + public Long getId() { + return id; + } + public void setId(Long id) { + this.id = id; + } + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public String getCode() { + return code; + } + public void setCode(String code) { + this.code = code; + } + public String getDescription() { + return description; + } + public void setDescription(String description) { + this.description = description; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardHeaderDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardHeaderDto.java new file mode 100644 index 000000000..33db8b1a2 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/DashboardHeaderDto.java @@ -0,0 +1,28 @@ +package com.tarento.analytics.dto; + +import com.tarento.analytics.model.LineData; + +public class DashboardHeaderDto { + + private String type; + private Object data; + public String getType() { + return type; + } + public void setType(String type) { + this.type = type; + } + public Object getData() { + return data; + } + public void setData(Object data) { + this.data = data; + } + public LineData getLineData() { + return lineData; + } + public void setLineData(LineData lineData) { + this.lineData = lineData; + } + private LineData lineData; +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Data.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Data.java new file mode 100644 index 000000000..ebd23aef8 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Data.java @@ -0,0 +1,57 @@ +package com.tarento.analytics.dto; + +import com.google.gson.JsonObject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Data { + + private String headerName; + private Object headerValue; + private String headerSymbol; + + public Data(String name, Object value, String symbol) { + this.headerName = name; + this.headerValue = value; + this.headerSymbol = symbol; + } + + public Data(String name, Object value, String symbol, List plots) { + this.headerName = name; + this.headerValue = value; + this.headerSymbol = symbol; + this.plots = plots; + } + + private List plots = new ArrayList<>(); + + public List getPlots() { + return plots; + } + + public void setPlots(List plots) { + this.plots = plots; + } + + public void setHeaderValue(Object headerValue) { + this.headerValue = headerValue; + } + + public String getHeaderName() { + return headerName; + } + + public Object getHeaderValue() { + return headerValue; + } + + public String getHeaderSymbol() { + return headerSymbol; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/MappingDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/MappingDto.java new file mode 100644 index 000000000..1bb98d2d1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/MappingDto.java @@ -0,0 +1,14 @@ +package com.tarento.analytics.dto; + +public class MappingDto { + + private Object mappings; + + public Object getMappings() { + return mappings; + } + + public void setMappings(Object mappings) { + this.mappings = mappings; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Plot.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Plot.java new file mode 100644 index 000000000..05b7e0ec6 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/Plot.java @@ -0,0 +1,40 @@ +package com.tarento.analytics.dto; + +import java.math.BigDecimal; + +public class Plot { + + private String label; + private String name; + private Double value; + private String symbol; + + public Plot(String name, Double value, String symbol) { + this.name = name; + this.value = value; + this.symbol = symbol; + } + + public String getName() { + return name; + } + + public Double getValue() { + return value; + } + + public String getSymbol() { + return symbol; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + public void setValue(Double value) { + this.value = value; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDate.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDate.java new file mode 100644 index 000000000..045f3520d --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDate.java @@ -0,0 +1,26 @@ +package com.tarento.analytics.dto; + +public class RequestDate { + private String targetDate; + private String startDate; + private String endDate; + public String getTargetDate() { + return targetDate; + } + public void setTargetDate(String targetDate) { + this.targetDate = targetDate; + } + public String getStartDate() { + return startDate; + } + public void setStartDate(String startDate) { + this.startDate = startDate; + } + public String getEndDate() { + return endDate; + } + public void setEndDate(String endDate) { + this.endDate = endDate; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDtoV2.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDtoV2.java new file mode 100644 index 000000000..12df6d59b --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDtoV2.java @@ -0,0 +1,22 @@ +package com.tarento.analytics.dto; + +import java.util.Map; + +public class RequestDtoV2 { + + private Map headers; + private AggregateRequestDtoV2 aggregationRequestDto; + + public Map getHeaders() { + return headers; + } + public void setHeaders(Map headers) { + this.headers = headers; + } + public AggregateRequestDtoV2 getAggregationRequestDto() { + return aggregationRequestDto; + } + public void setAggregationRequestDto(AggregateRequestDtoV2 aggregationRequestDto) { + this.aggregationRequestDto = aggregationRequestDto; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/ResponseDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/ResponseDto.java new file mode 100644 index 000000000..13ac13569 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/ResponseDto.java @@ -0,0 +1,40 @@ +package com.tarento.analytics.dto; + +import org.springframework.http.HttpStatus; + +public class ResponseDto { + + private int statusCode; + private String statusMessage; + private Object response; + + public ResponseDto(int statusCode, String statusMessage, Object response) { + this.statusCode = statusCode; + this.statusMessage = statusMessage; + this.response = response; + } + + public int getStatusCode() { + return statusCode; + } + + public void setStatusCode(int statusCode) { + this.statusCode = statusCode; + } + + public String getStatusMessage() { + return statusMessage; + } + + public void setStatusMessage(String statusMessage) { + this.statusMessage = statusMessage; + } + + public Object getResponse() { + return response; + } + + public void setResponse(Object response) { + this.response = response; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RoleDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RoleDto.java new file mode 100644 index 000000000..85fee2ee5 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/RoleDto.java @@ -0,0 +1,46 @@ +package com.tarento.analytics.dto; + +public class RoleDto { + + public Long id; + + public String description; + + public String name; + + public String orgId; + + public boolean isSuperAdmin; + + public String getOrgId() { + return orgId; + } + + public void setOrgId(String orgId) { + this.orgId = orgId; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/SearchDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/SearchDto.java new file mode 100644 index 000000000..14687f398 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/SearchDto.java @@ -0,0 +1,165 @@ +package com.tarento.analytics.dto; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This class will take input for elastic search query + * + * @author + */ +public class SearchDto { + + @SuppressWarnings("rawtypes") + private List properties; + + private List> facets = new ArrayList<>(); + private List fields; + private List excludedFields; + private Map sortBy = new HashMap<>(); + private String operation; + private String query; + private Integer limit = 250; + private Integer offset = 0; + private boolean fuzzySearch = false; + // additional properties will hold , filters, exist , not exist + private Map additionalProperties = new HashMap<>(); + private Map softConstraints = new HashMap<>(); + private List> groupQuery = new ArrayList<>(); + private List mode = new ArrayList<>(); + + public List> getGroupQuery() { + return groupQuery; + } + + public void setGroupQuery(List> groupQuery) { + this.groupQuery = groupQuery; + } + + public SearchDto() { + super(); + } + + @SuppressWarnings("rawtypes") + public SearchDto(List properties, String operation, int limit) { + super(); + this.properties = properties; + this.operation = operation; + this.limit = limit; + } + + @SuppressWarnings("rawtypes") + public List getProperties() { + return properties; + } + + @SuppressWarnings("rawtypes") + public void setProperties(List properties) { + this.properties = properties; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + public List> getFacets() { + return facets; + } + + public void setFacets(List> facets) { + this.facets = facets; + } + + public Map getSortBy() { + return sortBy; + } + + public void setSortBy(Map sortBy) { + this.sortBy = sortBy; + } + + public boolean isFuzzySearch() { + return fuzzySearch; + } + + public void setFuzzySearch(boolean fuzzySearch) { + this.fuzzySearch = fuzzySearch; + } + + public Map getAdditionalProperties() { + return additionalProperties; + } + + public void setAdditionalProperties(Map additionalProperties) { + this.additionalProperties = additionalProperties; + } + + public Object getAdditionalProperty(String key) { + return additionalProperties.get(key); + } + + public void addAdditionalProperty(String key, Object value) { + this.additionalProperties.put(key, value); + } + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + public Integer getOffset() { + return offset; + } + + public void setOffset(Integer offset) { + this.offset = offset; + } + + public Map getSoftConstraints() { + return softConstraints; + } + + public void setSoftConstraints(Map softConstraints) { + this.softConstraints = softConstraints; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public List getMode() { + return mode; + } + + public void setMode(List mode) { + this.mode = mode; + } + + public List getExcludedFields() { + return excludedFields; + } + + public void setExcludedFields(List excludedFields) { + this.excludedFields = excludedFields; + } +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/dto/UserDto.java b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/UserDto.java new file mode 100644 index 000000000..e548e8b4c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/dto/UserDto.java @@ -0,0 +1,69 @@ +package com.tarento.analytics.dto; + +import java.util.List; + +public class UserDto { + + private long id; + + private String userName; + + private String emailId; + + private List roles; + + private String orgId; + + private String countryCode; + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getEmailId() { + return emailId; + } + + public void setEmailId(String emailId) { + this.emailId = emailId; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public String getOrgId() { + return orgId; + } + + public void setOrgId(String orgId) { + this.orgId = orgId; + } + + public String getCountryCode() { + return countryCode; + } + + public void setCountryCode(String countryCode) { + this.countryCode = countryCode; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/enums/ChartType.java b/dashboard-analytics/src/main/java/com/tarento/analytics/enums/ChartType.java new file mode 100644 index 000000000..24bfb95e8 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/enums/ChartType.java @@ -0,0 +1,31 @@ +package com.tarento.analytics.enums; + +import org.apache.commons.lang3.StringUtils; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +public enum ChartType { + + TABLE("table"), PERFORM("perform"), METRIC("metric"), PIE("pie"), LINE("line"); + + private String value; + + ChartType(final String value) { + this.value = value; + } + + + @JsonValue + public String toString() { + return value; + } + + @JsonCreator + public static ChartType fromValue(final String passedValue) { + for (final ChartType obj : ChartType.values()) + if (String.valueOf(obj.value).toLowerCase().equals(passedValue.toLowerCase())) + return obj; + return null; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/exception/AINException.java b/dashboard-analytics/src/main/java/com/tarento/analytics/exception/AINException.java new file mode 100644 index 000000000..6e86721c3 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/exception/AINException.java @@ -0,0 +1,29 @@ +package com.tarento.analytics.exception; + +public class AINException extends Exception{ + + + //TODO- wrapped up with other exceptions. + private static final long serialVersionUID = 1L; + + String errorCode; + + String errorMessage; + + public AINException(String errorCode, String errorMessage) { + this.errorCode= errorCode; + this.errorMessage = errorMessage; + } + + public String getErrorCode() { + return errorCode; + } + + + public String getErrorMessage() { + return errorMessage; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/IResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/IResponseHandler.java new file mode 100644 index 000000000..d5893b993 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/IResponseHandler.java @@ -0,0 +1,149 @@ +package com.tarento.analytics.handler; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import com.tarento.analytics.enums.ChartType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Handles Elastic search consolidate responses + */ +public interface IResponseHandler { + + public static final Logger logger = LoggerFactory.getLogger(IResponseHandler.class); + + public static final String API_CONFIG_JSON = "ChartApiConfig.json"; + public static final String AGGS_PATH = "aggregationPaths"; + + public static final String CHART_NAME = "chartName"; + public static final String CHART_TYPE = "chartType"; + public static final String DRILL_CHART = "drillChart"; + public static final String VALUE_TYPE = "valueType"; + public static final String FILTER_KEYS = "filterKeys"; + + public final String ASC = "asc"; + public final String DESC = "desc"; + public final String RANK = "Rank"; + public final String AGGREGATIONS = "aggregations"; + public final String PLOT_LABEL = "plotLabel"; + public final String LIMIT = "limit"; + public final String ORDER = "order"; + public final String ACTION = "action"; + public final String TYPE_MAPPING = "pathDataTypeMapping"; + + public static String BUCKETS = "buckets"; + public static String KEY = "key"; + public static String VALUE = "value"; + + public static Double BOUNDARY_VALUE = 50.0; + + /** + * Translate the consolidated/aggregated response + * + * @param chartId + * @param aggregations + * @return + * @throws IOException + */ + public AggregateDto translate(String chartId, ObjectNode aggregations) throws IOException; + + /** + * Prepare aggregated dato for a chart node + * + * @param chartNode + * @param dataList - data plots object + * @return + */ + default AggregateDto getAggregatedDto(JsonNode chartNode, List dataList) { + AggregateDto aggregateDto = new AggregateDto(); + aggregateDto.setDrillDownChartId(chartNode.get(DRILL_CHART).asText()); + ChartType chartType = ChartType.fromValue(chartNode.get(CHART_TYPE).asText()); + aggregateDto.setChartType(chartType); + aggregateDto.setData(dataList); + if(null!=chartNode.get(FILTER_KEYS)) + aggregateDto.setFilter((ArrayNode) chartNode.get(FILTER_KEYS)); + return aggregateDto; + } + + /** + * Append computed field for a given Data, for its existing fields + * computes as partfield/wholefield * 100 + * + * @param data + * @param newfield + * @param partField + * @param wholeField + */ + default void addComputedField(Data data, String newfield, String partField, String wholeField) { + try { + Map plotMap = data.getPlots().stream().parallel().collect(Collectors.toMap(Plot::getName, Function.identity())); + + if (plotMap.get(partField).getValue() == 0.0 || plotMap.get(wholeField).getValue() == 0.0) { + data.getPlots().add(new Plot(newfield, 0.0, "percentage")); + } else { + double fieldValue = plotMap.get(partField).getValue() / plotMap.get(wholeField).getValue() * 100; + data.getPlots().add(new Plot(newfield, fieldValue, "percentage")); + + } + } catch (Exception e) { + e.printStackTrace(); + data.getPlots().add(new Plot(newfield, 0.0, "percentage")); + } + + } + + /** + * Computes the percentage from 0th and 1st index of list + * Ex: 0th element/1st element * 100 + * @param values + * @return + */ + default Double percentageValue(List values) { + double val = (values.get(0)/values.get(1) * 100); + return (values.size() > 1 && values.get(0) != 0.0 && values.get(1) != 0.0) ? val : 0.0; + } + + + /** + * Computes the percentage from 1st & 2nd element of collection + * Ex: first element/second element * 100 + * @param values + * @return + */ + default Double getPercentage(Map values, String partField, String wholeField) { + + double val = (values.get(partField)/ values.get(wholeField) * 100); + return (values.size() > 1 && values.get(partField) != 0.0 && values.get(wholeField) != 0.0) ? val: 0.0; + } + + /** + * Adding missing plot elements to Data + * @param plotKeys - all required plot key + * @param data + * @param symbol + */ + default void appendMissingPlot(Set plotKeys, Data data, String symbol) { + + Map map = data.getPlots().stream().collect(Collectors.toMap(Plot::getName, Function.identity())); + plotKeys.removeAll(map.keySet()); + plotKeys.forEach(plKey -> { + map.put(plKey, new Plot(plKey, new Double("0"), symbol)); + }); + + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/LineChartResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/LineChartResponseHandler.java new file mode 100644 index 000000000..508c8f8c1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/LineChartResponseHandler.java @@ -0,0 +1,132 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import com.tarento.analytics.enums.ChartType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.time.*; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +/** + * This handles ES response for single index, multiple index to represent data as line chart + * Creates plots by merging/computing(by summation) index values for same key + * AGGS_PATH : this defines the path/key to be used to search the tree + * VALUE_TYPE : defines the data type for the value formed, this could be amount, percentage, number + * + */ +@Component +public class LineChartResponseHandler implements IResponseHandler { + public static final Logger logger = LoggerFactory.getLogger(LineChartResponseHandler.class); + + @Autowired + ConfigurationLoader configurationLoader; + + @Override + public AggregateDto translate(String chartId, ObjectNode aggregations) throws IOException { + + List dataList = new LinkedList<>(); + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + ObjectNode configNode = configurationLoader.get(API_CONFIG_JSON); + JsonNode chartNode = configNode.get(chartId); + String symbol = chartNode.get(IResponseHandler.VALUE_TYPE).asText(); + ArrayNode aggrsPaths = (ArrayNode) chartNode.get(IResponseHandler.AGGS_PATH); + Set plotKeys = new HashSet<>(); + + aggrsPaths.forEach(headerPath -> { + List aggrNodes = aggregationNode.findValues(headerPath.asText()); + + Map plotMap = new LinkedHashMap<>(); + List totalValues = new ArrayList<>(); + aggrNodes.stream().parallel().forEach(aggrNode -> { + if (aggrNode.findValues(IResponseHandler.BUCKETS).size() > 0) { + + ArrayNode buckets = (ArrayNode) aggrNode.findValues(IResponseHandler.BUCKETS).get(0); + buckets.forEach(bucket -> { + String bkey = bucket.findValue(IResponseHandler.KEY).asText(); + String key = getWeekOfMonthOfYear(bkey); + + plotKeys.add(key); + double value = bucket.findValue(IResponseHandler.VALUE).asDouble(); + plotMap.put(key, plotMap.get(key) == null ? new Double("0") + value : plotMap.get(key) + value); + totalValues.add(value); + }); + } + }); + List plots = plotMap.entrySet().stream().map(e -> new Plot(e.getKey(), e.getValue(), symbol)).collect(Collectors.toList()); + try{ + Data data = new Data(headerPath.asText(), (totalValues==null || totalValues.isEmpty()) ? 0.0 : totalValues.stream().reduce(0.0, Double::sum), symbol); + data.setPlots(plots); + dataList.add(data); + }catch (Exception e) { + logger.info(" Legend/Header "+headerPath.asText() +" exception occurred "+e.getMessage()); + } + + + }); + + dataList.forEach(data -> { + appendMissingPlot(plotKeys, data, symbol); + }); + return getAggregatedDto(chartNode, dataList); + } + + private String getWeekOfMonthOfYear(String epocString) { + try { + long epoch = Long.parseLong( epocString ); + Date expiry = new Date( epoch ); + Calendar cal = Calendar.getInstance(); + cal.setTime(expiry); + + String day = String.valueOf(cal.get(Calendar.DATE)); + String month = monthNames(cal.get(Calendar.MONTH)+1); + String dayMonth = day.concat("-").concat(month); + String year = ""+cal.get(Calendar.YEAR); + String weekMonth = "Week " + cal.get(Calendar.WEEK_OF_YEAR) + " : " + dayMonth;//+" of Month "+ (cal.get(Calendar.MONTH) + 1); + return dayMonth; + } catch (Exception e) { + return epocString; + } + } + + private String monthNames(int month) { + if(month == 1) + return "Jan"; + else if(month == 2) + return "Feb"; + else if(month == 3) + return "Mar"; + else if(month == 4) + return "Apr"; + else if(month == 5) + return "May"; + else if(month == 6) + return "Jun"; + else if(month == 7) + return "Jul"; + else if(month == 8) + return "Aug"; + else if(month == 9) + return "Sep"; + else if(month == 10) + return "Oct"; + else if(month == 11) + return "Nov"; + else if(month == 12) + return "Dec"; + else + return "Month"; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricChartResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricChartResponseHandler.java new file mode 100644 index 000000000..7b1ad4f4a --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/MetricChartResponseHandler.java @@ -0,0 +1,73 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.Data; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * This handles ES response for single index, multiple index to represent single data value + * Creates plots by merging/computing(by summation or by percentage) index values for same key + * ACTION: for the chart config defines the type either summation or computing percentage + * AGGS_PATH : this defines the path/key to be used to search the tree + * + */ +@Component +public class MetricChartResponseHandler implements IResponseHandler{ + public static final Logger logger = LoggerFactory.getLogger(MetricChartResponseHandler.class); + + @Autowired + ConfigurationLoader configurationLoader; + public final String PERCENTAGE = "percentage"; + + + @Override + public AggregateDto translate(String chartId, ObjectNode aggregations) throws IOException { + List dataList = new ArrayList<>(); + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + JsonNode chartNode = configurationLoader.get(API_CONFIG_JSON).get(chartId); + + List totalValues = new ArrayList<>(); + String chartName = chartNode.get(CHART_NAME).asText(); + String action = chartNode.get(ACTION).asText(); + List percentageList = new ArrayList<>(); + + ArrayNode aggrsPaths = (ArrayNode) chartNode.get(AGGS_PATH); + + aggrsPaths.forEach(headerPath -> { + List values = aggregationNode.findValues(headerPath.asText()); + values.stream().parallel().forEach(value -> { + List valueNodes = value.findValues(VALUE).isEmpty() ? value.findValues("doc_count") : value.findValues(VALUE); + Double sum = valueNodes.stream().mapToDouble(o -> o.asDouble()).sum(); + if(action.equals(PERCENTAGE)){ + percentageList.add(sum); + } else { + totalValues.add(sum); + } + }); + }); + + String symbol = chartNode.get(IResponseHandler.VALUE_TYPE).asText(); + try{ + Data data = new Data(chartName, action.equals(PERCENTAGE)? percentageValue(percentageList) : (totalValues==null || totalValues.isEmpty())? 0.0 :totalValues.stream().reduce(0.0, Double::sum), symbol); + dataList.add(data); + }catch (Exception e){ + + logger.info("data chart name = "+chartName +" ex occurred "+e.getMessage()); + } + + return getAggregatedDto(chartNode, dataList); + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PerformanceChartResponeHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PerformanceChartResponeHandler.java new file mode 100644 index 000000000..e6625d42e --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PerformanceChartResponeHandler.java @@ -0,0 +1,118 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +@Component +public class PerformanceChartResponeHandler implements IResponseHandler { + + public static final Logger logger = LoggerFactory.getLogger(PerformanceChartResponeHandler.class); + + @Autowired + ConfigurationLoader configurationLoader; + + @Autowired + ObjectMapper mapper ; + + + @Override + public AggregateDto translate(String chartId, ObjectNode aggregations) throws IOException { + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + ObjectNode configNode = configurationLoader.get(API_CONFIG_JSON); + JsonNode chartNode = configNode.get(chartId); + String symbol = chartNode.get(IResponseHandler.VALUE_TYPE).asText(); + String plotLabel = chartNode.get(PLOT_LABEL).asText(); + String order = chartNode.get(ORDER).asText(); + int limit = chartNode.get(LIMIT).asInt(); + + + ArrayNode aggrsPaths = (ArrayNode) configNode.get(chartId).get(IResponseHandler.AGGS_PATH); + Map> mappings = new LinkedHashMap<>();//HashMap<>(); + + aggrsPaths.forEach(headerPath -> { + aggregationNode.findValues(headerPath.asText()).forEach(aggrNode -> { + if (aggrNode.findValues(IResponseHandler.BUCKETS).size() > 0) { + + ArrayNode buckets = (ArrayNode) aggrNode.findValues(IResponseHandler.BUCKETS).get(0); + buckets.forEach(bucket -> { + String key = bucket.findValue(IResponseHandler.KEY).asText(); + Double value = bucket.findValue(IResponseHandler.VALUE).asDouble(); + + if (mappings.containsKey(key)) { + Double sum = (mappings.get(key)).containsKey(headerPath.asText()) ? (mappings.get(key)).get(headerPath.asText()) + value : value; + (mappings.get(key)).put(headerPath.asText(), sum); + + } else { + Map additiveMap = new HashMap<>(); + additiveMap.put(aggrsPaths.get(0).asText(), new Double("0")); + additiveMap.put(aggrsPaths.get(1).asText(), new Double("0")); + + additiveMap.put(headerPath.asText(), value); + mappings.put(key, additiveMap); + } + }); + } + }); + }); + logger.info("performance chart data mappings : "+mappings); + List plotList = mappings.entrySet().stream().parallel().map(e -> new Plot(e.getKey(), getPercentage(e.getValue(), aggrsPaths.get(0).asText(),aggrsPaths.get(1).asText()), symbol)).collect(Collectors.toList()); + List plots = plotList.stream().filter(plot -> plot.getValue() != 0.0).collect(Collectors.toList()); + + plots.stream().parallel().forEach(item -> item.setLabel(plotLabel)); + Comparator plotValueComparator = Comparator.comparing(Plot::getValue); + plots.sort(plotValueComparator.reversed()); + return getAggregatedDto(chartNode, getDataOnPerformingOrder(plots, limit, order, symbol)); + } + + /** + * Prepare the plots with it's header Data in performing order + * @param plots + * @param limit n number of plots elements + * @param order top wise or bottom wise performance + * @param symbol + * @return + */ + private List getDataOnPerformingOrder(List plots, int limit, String order, String symbol) { + + List dataList = new ArrayList<>(); + if (order.equals(ASC)) { + for (int i = (plots.size() - 1); i >= 0; i--) { + dataList.add(getRankedPLot(i, symbol, plots)); + } + } else if (order.equals(DESC)) { + for (int i = 0; i < plots.size(); i++) { + dataList.add(getRankedPLot(i, symbol, plots)); + } + } + // return dataList.subList(Math.max(0, 0), Math.min(dataList.size(), limit)); + return dataList; + + } + + private Data getRankedPLot(int rank, String dataType, List plots){ + Data data = new Data(RANK, rank+1, dataType); + List p = new ArrayList(); + p.add(plots.get(rank)); + data.setPlots(p); + return data; + } + +} + + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PieChartResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PieChartResponseHandler.java new file mode 100644 index 000000000..93bc8523b --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/PieChartResponseHandler.java @@ -0,0 +1,78 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; +import com.tarento.analytics.enums.ChartType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +/** + * This handles ES response for single index, multiple index to represent data as pie figure + * Creates plots by merging/computing(by summation) index values for same key + * AGGS_PATH : this defines the path/key to be used to search the tree + * VALUE_TYPE : defines the data type for the value formed, this could be amount, percentage, number + * + */ +@Component +public class PieChartResponseHandler implements IResponseHandler { + public static final Logger logger = LoggerFactory.getLogger(PieChartResponseHandler.class); + + @Autowired + ConfigurationLoader configurationLoader; + @Autowired + ObjectMapper mapper; + + @Override + public AggregateDto translate(String chartId, ObjectNode aggregations) throws IOException { + + List dataList = new ArrayList<>(); + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + JsonNode chartNode = configurationLoader.get(API_CONFIG_JSON).get(chartId); + String headerKey = chartNode.get(CHART_NAME).asText(); + List headerPlotList = new ArrayList<>(); + List totalValue = new ArrayList<>(); + + String symbol = chartNode.get(IResponseHandler.VALUE_TYPE).asText(); + ArrayNode aggrsPaths = (ArrayNode) chartNode.get(IResponseHandler.AGGS_PATH); + + aggrsPaths.forEach(headerPath -> { + aggregationNode.findValues(headerPath.asText()).stream().parallel().forEach(valueNode->{ + if(valueNode.has(BUCKETS)){ + JsonNode buckets = valueNode.findValue(BUCKETS); + buckets.forEach(bucket -> { + totalValue.add(bucket.findValue(VALUE).asDouble()); + Plot plot = new Plot(bucket.findValue(KEY).asText(), bucket.findValue(VALUE).asDouble(), symbol); + headerPlotList.add(plot); + }); + + } else { + List valueNodes = valueNode.findValues(VALUE).isEmpty() ? valueNode.findValues("doc_count") : valueNode.findValues(VALUE); + double sum = valueNodes.stream().mapToLong(o -> o.asLong()).sum(); + totalValue.add(sum); + Plot plot = new Plot(headerPath.asText(), sum, symbol); + headerPlotList.add(plot); + } + }); + }); + + Data data = new Data(headerKey, totalValue.stream().reduce(0.0, Double::sum), symbol); + data.setPlots(headerPlotList); + dataList.add(data); + + return getAggregatedDto(chartNode, dataList); + + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/ResponseHandlerFactory.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/ResponseHandlerFactory.java new file mode 100644 index 000000000..d41185a85 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/ResponseHandlerFactory.java @@ -0,0 +1,44 @@ +package com.tarento.analytics.handler; + +import com.tarento.analytics.enums.ChartType; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class ResponseHandlerFactory { + + @Autowired + private MetricChartResponseHandler metricChartResponseHandler; + @Autowired + private LineChartResponseHandler lineChartResponseHandler; + @Autowired + private PieChartResponseHandler pieChartResponseHandler; + @Autowired + private PerformanceChartResponeHandler performingBarChartResponeHandler; + @Autowired + private TableChartResponseHandler tableChartResponseHandler; + + public IResponseHandler getInstance(ChartType chartType) { + + if (chartType == chartType.METRIC) { + return metricChartResponseHandler; + + } else if (chartType == chartType.LINE) { + return lineChartResponseHandler; + + } else if (chartType == chartType.PIE) { + return pieChartResponseHandler; + + } else if (chartType == chartType.PERFORM) { + return performingBarChartResponeHandler; + + } else if (chartType == chartType.TABLE) { + return tableChartResponseHandler; + } + + return null; + } + +} + + diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/handler/TableChartResponseHandler.java b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/TableChartResponseHandler.java new file mode 100644 index 000000000..d49bb11b2 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/handler/TableChartResponseHandler.java @@ -0,0 +1,102 @@ +package com.tarento.analytics.handler; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.Data; +import com.tarento.analytics.dto.Plot; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; +/** + * This handles ES response for single index, multiple index to compute performance + * Creates plots by performing ordered (ex: top n performance or last n performance) + * AGGS_PATH : configurable to this defines the path/key to be used to search the tree + * VALUE_TYPE : configurable to define the data type for the value formed, this could be amount, percentage, number + * PLOT_LABEL : configurable to define the label for the plot + * TYPE_MAPPING : defines for a plot data type + */ +@Component +public class TableChartResponseHandler implements IResponseHandler { + public static final Logger logger = LoggerFactory.getLogger(TableChartResponseHandler.class); + + @Autowired + ConfigurationLoader configurationLoader; + + private final String TOTAL_COLLECTION = "Total Collection"; + private final String TARGET_COLLECTION = "Target Collection"; + private final String TARGET_ACHIEVED = "Target Achieved"; + + + @Override + public AggregateDto translate(String chartId, ObjectNode aggregations) throws IOException { + + JsonNode aggregationNode = aggregations.get(AGGREGATIONS); + JsonNode chartNode = configurationLoader.get(API_CONFIG_JSON).get(chartId); + String plotLabel = chartNode.get(PLOT_LABEL).asText(); + ArrayNode pathDataTypeMap = (ArrayNode) chartNode.get(TYPE_MAPPING); + ArrayNode aggrsPaths = (ArrayNode) chartNode.get(IResponseHandler.AGGS_PATH); + Map> mappings = new HashMap<>(); + List aggrNodes = aggregationNode.findValues(BUCKETS); + + int[] idx = { 1 }; + + aggrNodes.stream().forEach(node -> { + ArrayNode buckets = (ArrayNode) node; + buckets.forEach(bucket -> { + Map plotMap = new LinkedHashMap<>(); + String key = bucket.findValue(IResponseHandler.KEY).asText(); + + aggrsPaths.forEach(headerPath -> { + JsonNode datatype = pathDataTypeMap.findValue(headerPath.asText()); + JsonNode valueNode = bucket.findValue(headerPath.asText()); + Double value = valueNode == null ? 0.0 : valueNode.get(VALUE).asDouble(); + Plot plot = new Plot(headerPath.asText(), value, datatype.asText()); + if (mappings.containsKey(key)) { + double newval = mappings.get(key).get(headerPath.asText()) == null ? value : (mappings.get(key).get(headerPath.asText()).getValue() + value); + plot.setValue(newval); + mappings.get(key).put(headerPath.asText(), plot); + } else { + plotMap.put(headerPath.asText(), plot); + } + }); + + if (plotMap.size() > 0) { + Map plots = new LinkedHashMap<>(); + Plot sno = new Plot("Sl no", null, "text"); + sno.setLabel("" + idx[0]++); + Plot plotkey = new Plot(plotLabel.isEmpty() ? "Key" : plotLabel, null, "text"); + plotkey.setLabel(key); + + plots.put("slno", sno); + plots.put(plotLabel.isEmpty() ? "Key" : plotLabel, plotkey); + plots.putAll(plotMap); + mappings.put(key, plots); + + } + }); + + }); + + List dataList = new ArrayList<>(); + mappings.entrySet().stream().parallel().forEach(plotMap -> { + List plotList = plotMap.getValue().values().stream().parallel().collect(Collectors.toList()); + Data data = new Data(plotMap.getKey(), new Integer(plotMap.getValue().get("slno").getLabel()), null); + data.setPlots(plotList); + addComputedField(data, TARGET_ACHIEVED, TOTAL_COLLECTION, TARGET_COLLECTION); + dataList.add(data); + + }); + dataList.sort((o1, o2) -> ((Integer) o1.getHeaderValue()).compareTo((Integer) o2.getHeaderValue())); + return getAggregatedDto(chartNode, dataList); + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/ElasticSearchDictator.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ElasticSearchDictator.java new file mode 100644 index 000000000..ec51d3054 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ElasticSearchDictator.java @@ -0,0 +1,69 @@ +package com.tarento.analytics.model; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * + * @author Darshan + * + */ +public class ElasticSearchDictator { + + private String visualisationName; + private String indexName; + private String documentType; + private Map>> queryMap; + private Map>>> aggregationMap; + private Map queryAggregationMap; + + public Map getQueryAggregationMap() { + return queryAggregationMap; + } + + public void setQueryAggregationMap(Map queryAggregationMap) { + this.queryAggregationMap = queryAggregationMap; + } + + public String getVisualisationName() { + return visualisationName; + } + + public void setVisualisationName(String visualisationName) { + this.visualisationName = visualisationName; + } + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getDocumentType() { + return documentType; + } + + public void setDocumentType(String documentType) { + this.documentType = documentType; + } + + public Map>> getQueryMap() { + return queryMap; + } + + public void setQueryMap(Map>> queryMap) { + this.queryMap = queryMap; + } + + public Map>>> getAggregationMap() { + return aggregationMap; + } + + public void setAggregationMap(Map>>> aggregationMap) { + this.aggregationMap = aggregationMap; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/Item.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Item.java new file mode 100644 index 000000000..b1e63e89e --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Item.java @@ -0,0 +1,92 @@ +package com.tarento.analytics.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Item { + + @JsonProperty("name") + private String name; + @JsonProperty("code") + private String code; + @JsonProperty("id") + private Long id; + @JsonProperty("description") + private String description; + @JsonProperty("price") + private Double price; + @JsonProperty("categoryId") + private Long categoryId; + @JsonProperty("subCategoryId") + private Long subCategoryId; + @JsonProperty("name") + + + public String getName() { + return name; + } + + public Long getCategoryId() { + return categoryId; + } + + public void setCategoryId(Long categoryId) { + this.categoryId = categoryId; + } + + public Long getSubCategoryId() { + return subCategoryId; + } + + public void setSubCategoryId(Long subCategoryId) { + this.subCategoryId = subCategoryId; + } + + @JsonProperty("name") + public void setName(String name) { + this.name = name; + } + + @JsonProperty("code") + public String getCode() { + return code; + } + + @JsonProperty("code") + public void setCode(String code) { + this.code = code; + } + + @JsonProperty("id") + public Long getId() { + return id; + } + + @JsonProperty("id") + public void setId(Long id) { + this.id = id; + } + + @JsonProperty("description") + public String getDescription() { + return description; + } + + @JsonProperty("description") + public void setDescription(String description) { + this.description = description; + } + + @JsonProperty("price") + public Double getPrice() { + return price; + } + + @JsonProperty("price") + public void setPrice(Double price) { + this.price = price; + } + + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/KeyData.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/KeyData.java new file mode 100644 index 000000000..6a49dd3bd --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/KeyData.java @@ -0,0 +1,53 @@ + +package com.tarento.analytics.model; + +public class KeyData { + + + private Long id; + private Object key; + private String label; + + public Long getId() { + return id; + } + public void setId(Long id) { + this.id = id; + } + public Object getKey() { + return key; + } + public void setKey(Object key) { + this.key = key; + } + public String getLabel() { + return label; + } + public void setLabel(String label) { + this.label = label; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((key == null) ? 0 : key.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + KeyData other = (KeyData) obj; + if (key == null) { + if (other.key != null) + return false; + } else if (!key.equals(other.key)) + return false; + return true; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/Label.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Label.java new file mode 100644 index 000000000..d0ef0c8e9 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Label.java @@ -0,0 +1,19 @@ +package com.tarento.analytics.model; + +public class Label { + + private String xLabel; + private String yLabel; + public String getxLabel() { + return xLabel; + } + public void setxLabel(String xLabel) { + this.xLabel = xLabel; + } + public String getyLabel() { + return yLabel; + } + public void setyLabel(String yLabel) { + this.yLabel = yLabel; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/LineData.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/LineData.java new file mode 100644 index 000000000..532eeb814 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/LineData.java @@ -0,0 +1,20 @@ +package com.tarento.analytics.model; + +public class LineData { + + private String type; + private Object value; + public String getType() { + return type; + } + public void setType(String type) { + this.type = type; + } + public Object getValue() { + return value; + } + public void setValue(Object value) { + this.value = value; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/Query.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Query.java new file mode 100644 index 000000000..bf34cdbc0 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Query.java @@ -0,0 +1,26 @@ +package com.tarento.analytics.model; + +import java.util.Map; + +public class Query { + + private String dateFilterField; + private Map aggregation; + + public String getDateFilterField() { + return dateFilterField; + } + + public void setDateFilterField(String dateFilterField) { + this.dateFilterField = dateFilterField; + } + + public Map getAggregation() { + return aggregation; + } + + public void setAggregation(Map aggregation) { + this.aggregation = aggregation; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/ServiceQuery.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ServiceQuery.java new file mode 100644 index 000000000..f6fdd64ed --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/ServiceQuery.java @@ -0,0 +1,27 @@ +package com.tarento.analytics.model; + +public class ServiceQuery { + + private String indexName; + private String queryJson; + private String documentType; + public String getIndexName() { + return indexName; + } + public void setIndexName(String indexName) { + this.indexName = indexName; + } + public String getQueryJson() { + return queryJson; + } + public void setQueryJson(String queryJson) { + this.queryJson = queryJson; + } + public String getDocumentType() { + return documentType; + } + public void setDocumentType(String documentType) { + this.documentType = documentType; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/Transaction.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Transaction.java new file mode 100644 index 000000000..5e5b063df --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/Transaction.java @@ -0,0 +1,264 @@ +package com.tarento.analytics.model; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Transaction Object which gets posted to Elastic Search + * @author Darshan Nagesh + * + */ +public class Transaction { + + @JsonProperty("transactionId") + private Long transactionId; + @JsonProperty("receiptNo") + private String receiptNo; + @JsonProperty("transType") + private String transType; + @JsonProperty("storeCode") + private Long storeCode; + @JsonProperty("posTerminalNo") + private String posTerminalNo; + @JsonProperty("staffId") + private Long staffId; + @JsonProperty("transDate") + private String transDate; + @JsonProperty("transTime") + private String transTime; + @JsonProperty("customerNo") + private Long consumerNo; + @JsonProperty("salesType") + private String salesType; + @JsonProperty("netAmt") + private Double netAmt; + @JsonProperty("grossAmt") + private Double grossAmt; + @JsonProperty("payment") + private Double payment; + @JsonProperty("discountAmt") + private Double discountAmt; + @JsonProperty("costAmt") + private Double costAmt; + @JsonProperty("managerId") + private Long managerId; + @JsonProperty("isTraining") + private Boolean isTraining; + @JsonProperty("statementNo") + private String statementNo; + @JsonProperty("postingStatus") + private String postingStatus; + @JsonProperty("postStatementNo") + private String postStatementNo; + @JsonProperty("customerAgeGroup") + private String customerAgeGroup; + @JsonProperty("customerGender") + private String customerGender; + @JsonProperty("items") + private List items; + @JsonProperty("itemDetails") + private List itemDetails; + @JsonProperty("orgId") + private Long orgId; + + public Long getOrgId() { + return orgId; + } + + public void setOrgId(Long orgId) { + this.orgId = orgId; + } + + public List getItemDetails() { + return itemDetails; + } + + public void setItemDetails(List itemDetails) { + this.itemDetails = itemDetails; + } + + public List getItems() { + return items; + } + + public void setItems(List items) { + this.items = items; + } + + public String getCustomerAgeGroup() { + return customerAgeGroup; + } + + public void setCustomerAgeGroup(String customerAgeGroup) { + this.customerAgeGroup = customerAgeGroup; + } + + public String getCustomerGender() { + return customerGender; + } + + public void setCustomerGender(String customerGender) { + this.customerGender = customerGender; + } + + public Long getTransactionId() { + return transactionId; + } + + public void setTransactionId(Long transactionId) { + this.transactionId = transactionId; + } + + public String getReceiptNo() { + return receiptNo; + } + + public void setReceiptNo(String receiptNo) { + this.receiptNo = receiptNo; + } + + public String getTransType() { + return transType; + } + + public void setTransType(String transType) { + this.transType = transType; + } + + public Long getStoreCode() { + return storeCode; + } + + public void setStoreCode(Long storeCode) { + this.storeCode = storeCode; + } + + public String getPosTerminalNo() { + return posTerminalNo; + } + + public void setPosTerminalNo(String posTerminalNo) { + this.posTerminalNo = posTerminalNo; + } + + public Long getStaffId() { + return staffId; + } + + public void setStaffId(Long staffId) { + this.staffId = staffId; + } + + public String getTransDate() { + return transDate; + } + + public void setTransDate(String transDate) { + this.transDate = transDate; + } + + public String getTransTime() { + return transTime; + } + + public void setTransTime(String transTime) { + this.transTime = transTime; + } + + public Long getConsumerNo() { + return consumerNo; + } + + public void setConsumerNo(Long consumerNo) { + this.consumerNo = consumerNo; + } + + public String getSalesType() { + return salesType; + } + + public void setSalesType(String salesType) { + this.salesType = salesType; + } + + public Double getNetAmt() { + return netAmt; + } + + public void setNetAmt(Double netAmt) { + this.netAmt = netAmt; + } + + public Double getGrossAmt() { + return grossAmt; + } + + public void setGrossAmt(Double grossAmt) { + this.grossAmt = grossAmt; + } + + public Double getPayment() { + return payment; + } + + public void setPayment(Double payment) { + this.payment = payment; + } + + public Double getDiscountAmt() { + return discountAmt; + } + + public void setDiscountAmt(Double discountAmt) { + this.discountAmt = discountAmt; + } + + public Double getCostAmt() { + return costAmt; + } + + public void setCostAmt(Double costAmt) { + this.costAmt = costAmt; + } + + public Long getManagerId() { + return managerId; + } + + public void setManagerId(Long managerId) { + this.managerId = managerId; + } + + public Boolean getIsTraining() { + return isTraining; + } + + public void setIsTraining(Boolean isTraining) { + this.isTraining = isTraining; + } + + public String getStatementNo() { + return statementNo; + } + + public void setStatementNo(String statementNo) { + this.statementNo = statementNo; + } + + public String getPostingStatus() { + return postingStatus; + } + + public void setPostingStatus(String postingStatus) { + this.postingStatus = postingStatus; + } + + public String getPostStatementNo() { + return postStatementNo; + } + + public void setPostStatementNo(String postStatementNo) { + this.postStatementNo = postStatementNo; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Chart.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Chart.java new file mode 100644 index 000000000..9a83dbf1e --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Chart.java @@ -0,0 +1,108 @@ +package com.tarento.analytics.model.dashboardConfig; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Chart { + +@JsonProperty("id") +private Long id; +@JsonProperty("name") +private String name; +@JsonProperty("code") +private String code; +@JsonProperty("serviceApi") +private String serviceApi; +@JsonProperty("chartType") +private String chartType; +@JsonProperty("headersAvailable") +private Boolean headersAvailable; +@JsonProperty("filter") +private String filter; +@JsonProperty("headers") +private List
headers = null; + + +public String getFilter() { + return filter; +} + +public void setFilter(String filter) { + this.filter = filter; +} + +@JsonProperty("id") +public Long getId() { +return id; +} + +@JsonProperty("id") +public void setId(Long id) { +this.id = id; +} + +@JsonProperty("name") +public String getName() { +return name; +} + +@JsonProperty("name") +public void setName(String name) { +this.name = name; +} + +@JsonProperty("code") +public String getCode() { +return code; +} + +@JsonProperty("code") +public void setCode(String code) { +this.code = code; +} + +@JsonProperty("serviceApi") +public String getServiceApi() { +return serviceApi; +} + +@JsonProperty("serviceApi") +public void setServiceApi(String serviceApi) { +this.serviceApi = serviceApi; +} + +@JsonProperty("chartType") +public String getChartType() { +return chartType; +} + +@JsonProperty("chartType") +public void setChartType(String chartType) { +this.chartType = chartType; +} + +@JsonProperty("headersAvailable") +public Boolean getHeadersAvailable() { +return headersAvailable; +} + +@JsonProperty("headersAvailable") +public void setHeadersAvailable(Boolean headersAvailable) { +this.headersAvailable = headersAvailable; +} + +@JsonProperty("headers") +public List
getHeaders() { +return headers; +} + +@JsonProperty("headers") +public void setHeaders(List
headers) { +this.headers = headers; +} + + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Dashboard.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Dashboard.java new file mode 100644 index 000000000..f06271f76 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Dashboard.java @@ -0,0 +1,100 @@ +package com.tarento.analytics.model.dashboardConfig; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tarento.analytics.dto.DashboardDto; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Dashboard { + +@JsonProperty("id") +private Long id; +@JsonProperty("name") +private String name; +@JsonProperty("code") +private String code; +@JsonProperty("isActive") +private Boolean isActive; +@JsonProperty("description") +private String description; +@JsonProperty("placement") +private String placement; +@JsonProperty("visualizations") +private List visualizations = null; + +public Dashboard() {} +public Dashboard(DashboardDto dto) { + this.id = dto.getId(); + this.name = dto.getName(); + this.code = dto.getCode(); + this.description = dto.getDescription(); + this.placement = dto.getPlacement(); +} + +public String getPlacement() { + return placement; +} +public void setPlacement(String placement) { + this.placement = placement; +} +public Long getId() { + return id; +} + +public void setId(Long id) { + this.id = id; +} + +@JsonProperty("name") +public String getName() { +return name; +} + +@JsonProperty("name") +public void setName(String name) { +this.name = name; +} + +@JsonProperty("code") +public String getCode() { +return code; +} + +@JsonProperty("code") +public void setCode(String code) { +this.code = code; +} + +@JsonProperty("isActive") +public Boolean getIsActive() { +return isActive; +} + +@JsonProperty("isActive") +public void setIsActive(Boolean isActive) { +this.isActive = isActive; +} + +@JsonProperty("description") +public String getDescription() { +return description; +} + +@JsonProperty("description") +public void setDescription(String description) { +this.description = description; +} + +@JsonProperty("visualizations") +public List getVisualizations() { +return visualizations; +} + +@JsonProperty("visualizations") +public void setVisualizations(List visualizations) { +this.visualizations = visualizations; +} + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Header.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Header.java new file mode 100644 index 000000000..60572bbf9 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Header.java @@ -0,0 +1,55 @@ +package com.tarento.analytics.model.dashboardConfig; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Header { + +@JsonProperty("id") +private Long id; +@JsonProperty("label") +private String label; +@JsonProperty("data") +private String data; +@JsonProperty("field") +private String field; + + +public Long getId() { + return id; +} + +public void setId(Long id) { + this.id = id; +} + +public String getField() { + return field; +} + +public void setField(String field) { + this.field = field; +} + +@JsonProperty("label") +public String getLabel() { +return label; +} + +@JsonProperty("label") +public void setLabel(String label) { +this.label = label; +} + +@JsonProperty("data") +public String getData() { +return data; +} + +@JsonProperty("data") +public void setData(String data) { +this.data = data; +} + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Visualization.java b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Visualization.java new file mode 100644 index 000000000..f88286ff8 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/model/dashboardConfig/Visualization.java @@ -0,0 +1,114 @@ +package com.tarento.analytics.model.dashboardConfig; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Visualization { + +@JsonProperty("id") +private Long id; +@JsonProperty("name") +private String name; +@JsonProperty("description") +private String description; +@JsonProperty("chartType") +private String chartType; +@JsonProperty("charts") +private List charts = null; +private List storeId; +@JsonProperty("salesAreaCode") +private String salesAreaCode; +@JsonProperty("countryCode") +private String countryCode; + + +public List getStoreId() { + return storeId; +} + + +@JsonProperty("visualRank") +private Long visualRank; + + +public void setStoreId(List storeId) { + this.storeId = storeId; +} + +public String getSalesAreaCode() { + return salesAreaCode; +} + +public void setSalesAreaCode(String salesAreaCode) { + this.salesAreaCode = salesAreaCode; +} + +public String getCountryCode() { + return countryCode; +} + +public void setCountryCode(String countryCode) { + this.countryCode = countryCode; +} + +public Long getVisualRank() { + return visualRank; +} + +public void setVisualRank(Long visualRank) { + this.visualRank = visualRank; +} + +public String getChartType() { + return chartType; +} + +public void setChartType(String chartType) { + this.chartType = chartType; +} + +@JsonProperty("id") +public Long getId() { +return id; +} + +@JsonProperty("id") +public void setId(Long id) { +this.id = id; +} + +@JsonProperty("name") +public String getName() { +return name; +} + +@JsonProperty("name") +public void setName(String name) { +this.name = name; +} + +@JsonProperty("description") +public String getDescription() { +return description; +} + +@JsonProperty("description") +public void setDescription(String description) { +this.description = description; +} + +@JsonProperty("charts") +public List getCharts() { +return charts; +} + +@JsonProperty("charts") +public void setCharts(List charts) { +this.charts = charts; +} + + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientService.java b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientService.java new file mode 100644 index 000000000..67cf66489 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/ClientService.java @@ -0,0 +1,14 @@ +package com.tarento.analytics.org.service; + +import java.io.IOException; +import java.util.List; + +import com.tarento.analytics.dto.*; +import com.tarento.analytics.exception.AINException; + +public interface ClientService { + + public AggregateDto getAggregatedData(AggregateRequestDtoV2 req, List roles) throws AINException, IOException; + public List getHeaderData(CummulativeDataRequestDto requestDto, List roles) throws AINException; + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/TarentoServiceImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/TarentoServiceImpl.java new file mode 100644 index 000000000..46fc0fbd0 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/org/service/TarentoServiceImpl.java @@ -0,0 +1,95 @@ +package com.tarento.analytics.org.service; + +import java.io.IOException; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.dto.AggregateDto; +import com.tarento.analytics.dto.AggregateRequestDtoV2; +import com.tarento.analytics.dto.CummulativeDataRequestDto; +import com.tarento.analytics.dto.DashboardHeaderDto; +import com.tarento.analytics.dto.RoleDto; +import com.tarento.analytics.enums.ChartType; +import com.tarento.analytics.exception.AINException; +import com.tarento.analytics.handler.IResponseHandler; +import com.tarento.analytics.handler.ResponseHandlerFactory; +import com.tarento.analytics.service.QueryService; +import com.tarento.analytics.service.impl.QueryServiceTemplate; + + +@Component +public class TarentoServiceImpl implements ClientService { + + public static final Logger logger = LoggerFactory.getLogger(TarentoServiceImpl.class); + + @Autowired + private QueryService queryService; + + @Autowired + private QueryServiceTemplate queryServiceTemplate; + + @Autowired + private ConfigurationLoader configurationLoader; + + @Autowired + private ResponseHandlerFactory responseHandlerFactory; + + @Override + public AggregateDto getAggregatedData(AggregateRequestDtoV2 request, List roles) throws AINException, IOException { + // Read visualization Code + String chartId = request.getVisualizationCode(); + + ObjectNode aggrObjectNode = JsonNodeFactory.instance.objectNode(); + ObjectNode nodes = JsonNodeFactory.instance.objectNode(); + + // Load Chart API configuration to Object Node for easy retrieval later + ObjectNode node = configurationLoader.get(Constants.ConfigurationFiles.CHART_API_CONFIG); + ChartType chartType = ChartType.fromValue(node.get(chartId).get(Constants.JsonPaths.CHART_TYPE).asText()); + ArrayNode queries = (ArrayNode) node.get(chartId).get(Constants.JsonPaths.QUERIES); + queries.forEach(query -> { + String module = query.get(Constants.JsonPaths.MODULE).asText(); + if(request.getModuleLevel().equals(Constants.Modules.HOME_REVENUE) || + request.getModuleLevel().equals(Constants.Modules.HOME_SERVICES) || + query.get(Constants.JsonPaths.MODULE).asText().equals(Constants.Modules.COMMON) || + request.getModuleLevel().equals(module)) { + + String indexName = query.get(Constants.JsonPaths.INDEX_NAME).asText(); + ObjectNode objectNode = queryService.getChartConfigurationQuery(request, query, indexName); + try { + JsonNode aggrNode = queryServiceTemplate.search(indexName,objectNode.toString()); + nodes.set(indexName,aggrNode.get(Constants.JsonPaths.AGGREGATIONS)); + }catch (Exception e) { + logger.error("Encountered an Exception while Executing the Query : " + e.getMessage()); + } + aggrObjectNode.set(Constants.JsonPaths.AGGREGATIONS, nodes); + + } + }); + IResponseHandler responseHandler = responseHandlerFactory.getInstance(chartType); + AggregateDto aggregateDto = new AggregateDto(); + if(aggrObjectNode.fields().hasNext()){ + aggregateDto = responseHandler.translate(chartId, aggrObjectNode); + } + return aggregateDto; + } + + @Override + public List getHeaderData(CummulativeDataRequestDto requestDto, List roles) throws AINException { + // TODO Auto-generated method stub + return null; + } + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Aggregation.java b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Aggregation.java new file mode 100644 index 000000000..b7f4c8a28 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Aggregation.java @@ -0,0 +1,5 @@ +package com.tarento.analytics.query.model; + +public class Aggregation { + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Query.java b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Query.java new file mode 100644 index 000000000..d48b10b25 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/Query.java @@ -0,0 +1,28 @@ +package com.tarento.analytics.query.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Query { + + @JsonProperty("x_axis") + public XAxis xAxis; + + @JsonProperty("y_axis") + public YAxis yAxis; + + public XAxis getxAxis() { + return xAxis; + } + + public void setxAxis(XAxis xAxis) { + this.xAxis = xAxis; + } + + public YAxis getyAxis() { + return yAxis; + } + + public void setyAxis(YAxis yAxis) { + this.yAxis = yAxis; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/XAxis.java b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/XAxis.java new file mode 100644 index 000000000..e7b5a9bed --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/XAxis.java @@ -0,0 +1,18 @@ +package com.tarento.analytics.query.model; + +import java.util.Map; + +public class XAxis { + + public Map aggregation; + + public Map getAggregation() { + return aggregation; + } + + public void setAggregation(Map aggregation) { + this.aggregation = aggregation; + } + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/YAxis.java b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/YAxis.java new file mode 100644 index 000000000..e44af59a1 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/query/model/YAxis.java @@ -0,0 +1,17 @@ +package com.tarento.analytics.query.model; + +import java.util.Map; + +public class YAxis { + + Map aggregation; + + public Map getAggregation() { + return aggregation; + } + + public void setAggregation(Map aggregation) { + this.aggregation = aggregation; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/repository/ElasticSearchRepository.java b/dashboard-analytics/src/main/java/com/tarento/analytics/repository/ElasticSearchRepository.java new file mode 100644 index 000000000..4da1f8de9 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/repository/ElasticSearchRepository.java @@ -0,0 +1,84 @@ +package com.tarento.analytics.repository; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RestTemplate; + +import com.tarento.analytics.dto.MappingDto; +import com.tarento.analytics.model.Transaction; + +/** + * This Repository Class is used to perform the transactions of storing the data into the Elastic Search Repository + * @author Darshan Nagesh + * + */ +@Service +public class ElasticSearchRepository { + + public static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchRepository.class); + + private final RestTemplate restTemplate; + + public ElasticSearchRepository(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + } + + /** + * Based on the Transaction Index Data Obtained and the URL with Headers, this method will put the Data obtained on the + * Elastic Search Database and returns the response in the form of Positive or Negative outcome (True Or False) + * @param transactionIndex + * @param url + * @param headers + * @return + */ + public Boolean saveTransaction(Transaction transaction, String url, HttpHeaders headers) { + ResponseEntity map = null; + try { + map = restTemplate.exchange(url, HttpMethod.PUT, + new HttpEntity<>(transaction, headers), Map.class); + } catch (final HttpClientErrorException httpClientErrorException) { + LOGGER.error("Error : " + httpClientErrorException); + } catch (HttpServerErrorException httpServerErrorException) { + LOGGER.error("Error : " + httpServerErrorException); + } catch (Exception e) { + LOGGER.error("Error : " + e); + } + if (map != null && map.getStatusCode() != null && (map.getStatusCode() == HttpStatus.OK) || (map.getStatusCode() == HttpStatus.CREATED)) { + return true; + } + return false; + } + + + + // create index + public Boolean createTransactionIndex(MappingDto dto, String url, HttpHeaders headers){ + ResponseEntity map = null; + try { + LOGGER.info("Each Channel Content : " + dto.toString()); + map = restTemplate.exchange(url, HttpMethod.PUT, + new HttpEntity<>(dto, headers), Map.class); + } catch (final HttpClientErrorException httpClientErrorException) { + LOGGER.error("Error : " + httpClientErrorException); + } catch (HttpServerErrorException httpServerErrorException) { + LOGGER.error("Error : " + httpServerErrorException); + } catch (Exception e) { + LOGGER.error("Error : " + e); + } + if (map != null && map.getStatusCode() != null && (map.getStatusCode() == HttpStatus.OK) || (map.getStatusCode() == HttpStatus.CREATED)) { + return true; + } + return false; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/MetadataService.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/MetadataService.java new file mode 100644 index 000000000..43b8bbad2 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/MetadataService.java @@ -0,0 +1,13 @@ +package com.tarento.analytics.service; + +import java.io.IOException; +import java.util.List; + +import com.tarento.analytics.dto.RoleDto; +import com.tarento.analytics.exception.AINException; + +public interface MetadataService { + + public Object getDashboardConfiguration(String dashboardId, String catagory, List roleIds) throws AINException, IOException; + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/QueryService.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/QueryService.java new file mode 100644 index 000000000..aaa662dbd --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/QueryService.java @@ -0,0 +1,30 @@ +package com.tarento.analytics.service; + +import java.io.IOException; + +import org.elasticsearch.search.aggregations.Aggregations; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.AggregateRequestDtoV2; +import com.tarento.analytics.exception.AINException; + +public interface QueryService { +/* + List getAggregateData(AggregateRequestDto aggregateDto, String orgId) throws AINException; +*/ + public static final String API_CONFIG_JSON = "ChartApiConfig.json"; + public static final String DATE_SOURCE_FIELD = "dateSourceField"; + public static final String AGG_QUERY_JSON = "aggregationQueryJson"; + public static final String INDEX_NAME = "indexName" ; + public static final String DOCUMENT_TYPE = "documentType"; +/* Aggregations getAggregateData(AggregateRequestDto aggregateDto, String orgId) throws AINException; + Aggregations getAggregateDataV2(AggregateRequestDtoV2 aggregateDto, String orgId) throws AINException, JsonParseException, JsonMappingException, IOException; + */ + ObjectNode getChartConfigurationQuery(AggregateRequestDtoV2 req, JsonNode query, String indexName); + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/MetadataServiceImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/MetadataServiceImpl.java new file mode 100644 index 000000000..95d47f114 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/MetadataServiceImpl.java @@ -0,0 +1,64 @@ +package com.tarento.analytics.service.impl; + +import java.io.IOException; +import java.util.List; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.tarento.analytics.dto.RoleDto; +import com.tarento.analytics.exception.AINException; +import com.tarento.analytics.service.MetadataService; + +@Service("metadataService") +public class MetadataServiceImpl implements MetadataService { + + public static final Logger logger = LoggerFactory.getLogger(MetadataServiceImpl.class); + + @Autowired + private ConfigurationLoader configurationLoader; + @Autowired + private ObjectMapper objectMapper; + + @Override + public ArrayNode getDashboardConfiguration(String dashboardId, String catagory, List roleIds) throws AINException, IOException { + + ObjectNode configNode = configurationLoader.get(ConfigurationLoader.ROLE_DASHBOARD_CONFIG); + ArrayNode rolesArray = (ArrayNode) configNode.findValue("roles"); + ArrayNode dbArray = JsonNodeFactory.instance.arrayNode(); + + rolesArray.forEach(role -> { + Object roleId = roleIds.stream().filter(x -> role.get("roleId").asLong() == (x.getId())).findAny().orElse(null); + System.out.println("roleId = "+roleId); + + if (null != roleId) { + role.get("dashboards").forEach(dashboard -> { + ObjectNode copyDashboard = dashboard.deepCopy(); + ArrayNode visArray = JsonNodeFactory.instance.arrayNode(); + if(catagory != null) { + copyDashboard.get("visualizations").forEach(visual ->{ + if(visual.get("name").asText().equalsIgnoreCase(catagory)){ + visArray.add(visual); + } + }); + copyDashboard.set("visualizations", visArray); + } + if(dashboard.get("id").asText().equalsIgnoreCase(dashboardId)){ + dbArray.add(copyDashboard); + } + }); + } + }); + + //List dbs = objectMapper.readValue(dbArray.toString(), new TypeReference>() {}); + return dbArray; + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceImpl.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceImpl.java new file mode 100644 index 000000000..40580535c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceImpl.java @@ -0,0 +1,457 @@ +package com.tarento.analytics.service.impl; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg; +import org.elasticsearch.search.aggregations.metrics.sum.ParsedSum; +import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tarento.analytics.ConfigurationLoader; +import com.tarento.analytics.constant.Constants; +import com.tarento.analytics.dao.ElasticSearchDao; +import com.tarento.analytics.dto.AggregateRequestDto; +import com.tarento.analytics.dto.AggregateRequestDtoV2; +import com.tarento.analytics.enums.ChartType; +import com.tarento.analytics.exception.AINException; +import com.tarento.analytics.model.ElasticSearchDictator; +import com.tarento.analytics.model.KeyData; +import com.tarento.analytics.model.Query; +import com.tarento.analytics.model.ServiceQuery; +import com.tarento.analytics.service.QueryService; +import com.tarento.analytics.utils.ElasticProperties; + +@Component +public class QueryServiceImpl implements QueryService { + + public static final Logger logger = LoggerFactory.getLogger(QueryServiceImpl.class); +/* + @Autowired + private MetadataDao metaDataDao;*/ + + @Autowired + private ElasticSearchDao elasticSearchDao; + +/* + @Autowired + private DecoratorService decoratorService; + +*/ + @Autowired + private ConfigurationLoader configurationLoader; + + private static final Map WeekDayMap = createMap(); + + private static Map createMap() { + Map result = new HashMap(); + result.put(1, "SUN"); + result.put(2, "MON"); + result.put(3, "TUE"); + result.put(4, "WED"); + result.put(5, "THU"); + result.put(6, "FRI"); + result.put(7, "SAT"); + + return Collections.unmodifiableMap(result); + } + +/* + @Override + public Aggregations getAggregateData(AggregateRequestDto aggregateDto, String orgId) throws AINException { + + //public List getAggregateData(AggregateRequestDto aggregateDto, String orgId) throws AINException { + Aggregations aggregations = null; + ElasticSearchDictator dictator = null; + ServiceQuery elasticServiceQuery = metaDataDao.getServiceQuery(Long.parseLong(orgId), + aggregateDto.getServiceApi()); + + ObjectMapper mapper = new ObjectMapper(); + */ +/*Map resonseMap = null;*//* + + try { + + //Query Builder + Query queryJson = mapper.readValue(elasticServiceQuery.getQueryJson(), Query.class); + // Set index name and document Type for elastic + Map queryMap = queryJson.getAggregation(); + + //Label Mapping + Map labelMap = new HashMap<>(); + getAggregateLabelRecursively(queryMap,labelMap); + + dictator = elasticSearchDao.createSearchDictator(aggregateDto, elasticServiceQuery.getIndexName(), + elasticServiceQuery.getDocumentType(), queryJson.getDateFilterField()); + dictator.setQueryAggregationMap(queryMap); + SearchRequest searchRequest = elasticSearchDao.buildElasticSearchQuery(dictator); + List searchRequestList = new ArrayList<>(); + searchRequestList.add(searchRequest); + MultiSearchResponse response = elasticSearchDao.executeMultiSearchRequest(searchRequestList, Boolean.TRUE); + SearchResponse searchResponse = response.getResponses()[0].getResponse(); + aggregations = searchResponse.getAggregations(); + */ +/*resonseMap= translateResponse(response, aggregateDto.getChartType(),aggregateDto.getInterval(), labelMap); + decoratorService.getChartData(aggregateDto, chartFormat, chartType, serviceApi, chartCode)*//* + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return aggregations; + } + + @Override + public Aggregations getAggregateDataV2(AggregateRequestDtoV2 aggregateDto, String orgId) throws AINException, JsonParseException, JsonMappingException, IOException { + Boolean primaryOrNot = Boolean.TRUE; + ServiceQuery elasticServiceQuery = new ServiceQuery(); + String visualizationCode = aggregateDto.getVisualizationCode(); + ObjectNode configNode = configurationLoader.get(API_CONFIG_JSON); + elasticServiceQuery.setIndexName(configNode.get(visualizationCode).get(INDEX_NAME).asText()); + elasticServiceQuery.setDocumentType(configNode.get(visualizationCode).get(DOCUMENT_TYPE).asText()); + elasticServiceQuery.setQueryJson(configNode.get(visualizationCode).get(AGG_QUERY_JSON).asText()); + if(elasticServiceQuery.getIndexName().equals("dss-col-v1")) { + primaryOrNot = Boolean.TRUE; + } else { + primaryOrNot = Boolean.FALSE; + } + ObjectMapper mapper = new ObjectMapper(); + Query queryJson = mapper.readValue(elasticServiceQuery.getQueryJson(), Query.class); + + // Set index name and document Type for elastic + Map queryMap = queryJson.getAggregation(); + + //Label Mapping + Map labelMap = new HashMap<>(); + getAggregateLabelRecursively(queryMap,labelMap); + + Aggregations aggregations = null; + ElasticSearchDictator dictator = null; + + try { + dictator = elasticSearchDao.createSearchDictatorV2(aggregateDto, elasticServiceQuery.getIndexName(), + elasticServiceQuery.getDocumentType(), queryJson.getDateFilterField()); + dictator.setQueryAggregationMap(queryMap); + SearchRequest searchRequest = elasticSearchDao.buildElasticSearchQuery(dictator); + List searchRequestList = new ArrayList<>(); + searchRequestList.add(searchRequest); + MultiSearchResponse response = elasticSearchDao.executeMultiSearchRequest(searchRequestList, primaryOrNot); + SearchResponse searchResponse = response.getResponses()[0].getResponse(); + aggregations = searchResponse.getAggregations(); + } catch (Exception e) { + logger.error("Encountered an error while getting the Aggregated Data : " + e.getMessage()); + } + return aggregations; + } +*/ + +/* private Map translateResponse(MultiSearchResponse response, ChartType chartType, String interval, Map labelMap) { + SearchResponse searchResponse = response.getResponses()[0].getResponse(); + List aggregations = searchResponse.getAggregations().asList(); + Map responseMap = new HashMap<>(); + + if (chartType.equals(ChartType.BAR) || chartType.equals(ChartType.HORIZONTALBAR) + || chartType.equals(ChartType.PIE) || chartType.equals(ChartType.DOUGHNUT)) { + + if (aggregations.get(0) instanceof ParsedLongTerms) { + responseMap = parseParseLongTerms((ParsedLongTerms) aggregations.get(0), chartType, labelMap); + } + } + + return responseMap; + }*/ + + @SuppressWarnings("unchecked") + void getAggregateLabelRecursively(Map queryMap, Map labelMap ){ + try { + if(queryMap.containsKey(ElasticProperties.Query.AGGREGATION_CONDITION.toLowerCase())){ + + Map valueMap =(HashMap)queryMap.get(ElasticProperties.Query.AGGREGATION_CONDITION.toLowerCase()); + getAggregateLabelRecursively(valueMap, labelMap); + } + for (Map.Entry itrQuery : queryMap.entrySet()) { + if(itrQuery.getKey().equals(ElasticProperties.Query.AGGREGATION_CONDITION.toLowerCase())){ + continue; + } + Map propertiesMap = (HashMap)itrQuery.getValue(); + labelMap.put(itrQuery.getKey(), propertiesMap.get(ElasticProperties.Query.LABEL.toLowerCase()).toString()); + } + } catch (Exception e) { + + } + } + /* + * private Map + * parseDateHistogramForMultiLine(List aggregations, ChartType + * chartType, String interval) { Map responseMap = new + * HashMap<>(); + * + * for (Histogram.Bucket bucket : ((ParsedDateHistogram) + * aggregations.get(0)).getBuckets()) { List subAggregations = + * bucket.getAggregations().asList(); if (subAggregations.get(0) instanceof + * ParsedLongTerms) { parseParseLongTermsOnInterval(bucket, chartType, + * interval, responseMap); } + * + * } + * + * return responseMap; + * + * } + */ + + /* + * private Map parseNested(List aggregations, + * ChartType chartType, String interval) { + * + * Map responseMap = new HashMap<>(); for (Aggregation + * aggregationData : ((ParsedNested) aggregations.get(0)).getAggregations()) + * { if (aggregationData instanceof ParsedLongTerms) { responseMap = + * parseParseLongTerms((ParsedLongTerms) aggregationData, chartType, + * interval); } + * + * } return responseMap; + * + * } + */ + + /* + * private Map parseDateHistogram(List + * aggregations, ChartType chartType, String interval) { + * + * Map responseMap = new HashMap<>(); if + * (interval.equals(ElasticSearchConstants.DAY_OF_WEEK)) { Map dayWiseObjectMap = new HashMap() { { put(1, + * null); put(2, null); put(3, null); put(4, null); put(5, null); put(6, + * null); put(7, null); + * + * } }; + * + * for (Histogram.Bucket bucket : ((ParsedDateHistogram) + * aggregations.get(0)).getBuckets()) { Calendar cal = + * Calendar.getInstance(); + * cal.setTimeInMillis(Long.parseLong(bucket.getKeyAsString())); int + * dayofWeek = cal.get(Calendar.DAY_OF_WEEK); + * + * Object val = getAggregationValue(bucket); if + * (dayWiseObjectMap.containsKey(dayofWeek)) { Object dayWiseCount = + * dayWiseObjectMap.get(dayofWeek); if (val instanceof Double) { Double + * doubleValue = 0.0; if (dayWiseCount == null) { doubleValue = 0.0; } else + * { doubleValue = (Double) dayWiseCount; } doubleValue += + * getFormattedDouble((Double) val); dayWiseCount = doubleValue; } else if + * (val instanceof Long) { Long longValue = 0L; if (dayWiseCount == null) { + * longValue = 0L; } else { longValue = (Long) dayWiseCount; } longValue += + * (Long) val; dayWiseCount = longValue; } dayWiseObjectMap.put(dayofWeek, + * dayWiseCount); } else { if (val instanceof Double) { Double doubleValue = + * (Double) val; doubleValue += getFormattedDouble((Double) val); val = + * doubleValue; } dayWiseObjectMap.put(dayofWeek, val); } } for + * (Map.Entry entry : dayWiseObjectMap.entrySet()) { + * + * responseMap.put(WeekDayMap.get(entry.getKey()), (entry.getValue())); } } + * else { for (Histogram.Bucket bucket : ((ParsedDateHistogram) + * aggregations.get(0)).getBuckets()) { List subAggregations = + * bucket.getAggregations().asList(); if (subAggregations.get(0) instanceof + * ParsedLongTerms) { + * + * + * responseMap = parseParseLongTermsOnInterval(bucket, chartType, interval); + * } else { responseMap = parseDateHistogramBasedOnInterval(bucket, + * chartType, interval); } + * + * } } return responseMap; + * + * } + */ + /* + * private void parseParseLongTermsOnInterval(Bucket buckets, ChartType + * chartType, String interval, Map responseMap) { Calendar + * cal = Calendar.getInstance(); + * cal.setTimeInMillis(Long.parseLong(buckets.getKeyAsString())); String key + * = null; if (interval.equals(ElasticSearchConstants.DAY)) { int day = + * cal.get(Calendar.DAY_OF_MONTH); int month = cal.get(Calendar.MONTH) + 1; + * + * key = day + "/" + month; } else if + * (interval.equals(ElasticSearchConstants.HOUR)) { int hour = + * cal.get(Calendar.HOUR_OF_DAY); String suffix = "AM"; if + * (cal.get(Calendar.AM_PM) > 0) suffix = "PM"; + * + * key = String.valueOf(hour + 1) + suffix; + * + * } Map innerResponseMap = new HashMap<>(); + * + * for (Terms.Bucket bucket : ((ParsedLongTerms) buckets).getBuckets()) { + * + * Map valueMap = bucket.getAggregations().getAsMap(); + * Object val = null; + * + * for (Map.Entry aggregation : valueMap.entrySet()) { + * if + * (aggregation.getKey().contains(ElasticProperties.Query.SUM.toLowerCase()) + * ) { ParsedSum sum = (ParsedSum) aggregation.getValue(); val = + * sum.getValue(); } else if + * (aggregation.getKey().contains(ElasticProperties.Query.AVG.toLowerCase()) + * ) { ParsedAvg avg = (ParsedAvg) aggregation.getValue(); val = + * avg.getValue(); + * + * } else if + * (aggregation.getKey().contains(ElasticProperties.Query.COUNT.toLowerCase( + * ))) { ParsedValueCount count = (ParsedValueCount) aggregation.getValue(); + * val = count.getValue(); + * + * } + * + * } innerResponseMap.put(bucket.getKeyAsString(), val); } + * responseMap.put(key, innerResponseMap); + * + * } + */ + + /* + * private Map parseDateHistogramBasedOnInterval(Bucket + * bucket, ChartType chartType, String interval) { Map + * responseMap = new HashMap<>(); Calendar cal = Calendar.getInstance(); + * cal.setTimeInMillis(Long.parseLong(bucket.getKeyAsString())); String key + * = null; Object val = null; if + * (interval.equals(ElasticSearchConstants.DAY)) { int day = + * cal.get(Calendar.DAY_OF_MONTH); int month = cal.get(Calendar.MONTH) + 1; + * + * key = day + "/" + month; val = getAggregationValue(bucket); + * + * } else if (interval.equals(ElasticSearchConstants.HOUR)) { int hour = + * cal.get(Calendar.HOUR_OF_DAY); String suffix = "AM"; if + * (cal.get(Calendar.AM_PM) > 0) suffix = "PM"; val = + * getAggregationValue(bucket); + * + * key = String.valueOf(hour + 1) + suffix; + * + * } responseMap.put(key, val); return responseMap; + * + * } + */ + + /* + * private Object getAggregationValue(Bucket bucket) { Map valueMap = bucket.getAggregations().getAsMap(); Object val = + * null; + * + * for (Map.Entry aggregation : valueMap.entrySet()) { + * if + * (aggregation.getKey().contains(ElasticProperties.Query.SUM.toLowerCase()) + * ) { ParsedSum sum = (ParsedSum) aggregation.getValue(); val = + * sum.getValue(); } else if + * (aggregation.getKey().contains(ElasticProperties.Query.AVG.toLowerCase()) + * ) { ParsedAvg avg = (ParsedAvg) aggregation.getValue(); val = + * avg.getValue(); + * + * } else if + * (aggregation.getKey().contains(ElasticProperties.Query.COUNT.toLowerCase( + * ))) { ParsedValueCount count = (ParsedValueCount) aggregation.getValue(); + * val = count.getValue(); + * + * } } return val; } + */ + + private Map parseParseLongTerms(ParsedLongTerms aggregations, ChartType chartType, Map labelMap) { + + Map keyValueMap = new HashMap<>(); + + for (Terms.Bucket bucket : aggregations.getBuckets()) { + KeyData keyData = new KeyData(); + KeyData valueData = new KeyData(); + + String key = bucket.getKey().toString(); + + Map valueMap = bucket.getAggregations().getAsMap(); + Object val = null; + String valueLabel = null; + for (Map.Entry aggregation : valueMap.entrySet()) { + if (aggregation.getKey().contains(ElasticProperties.Query.SUM.toLowerCase())) { + ParsedSum sum = (ParsedSum) aggregation.getValue(); + val = sum.getValue(); + valueLabel = ElasticProperties.Query.SUM.toLowerCase(); + } else if (aggregation.getKey().contains(ElasticProperties.Query.AVG.toLowerCase())) { + ParsedAvg avg = (ParsedAvg) aggregation.getValue(); + val = avg.getValue(); + valueLabel = ElasticProperties.Query.SUM.toLowerCase(); + + } else if (aggregation.getKey().contains(ElasticProperties.Query.COUNT.toLowerCase())) { + ParsedValueCount count = (ParsedValueCount) aggregation.getValue(); + val = count.getValue(); + valueLabel = ElasticProperties.Query.SUM.toLowerCase(); + + } + + } + keyData.setKey(key); + keyData.setLabel(labelMap.get(ElasticProperties.Query.TERM.toLowerCase())); + valueData.setKey(val); + valueData.setLabel(labelMap.get(valueLabel)); + keyValueMap.put(keyData, valueData); + + } + return keyValueMap; + + } + + private Double getFormattedDouble(double val) { + return (Math.round(new Double(val).isInfinite() ? 0.0 : new Double(val) * 100.0) / 100.0); + } + + @Override + public ObjectNode getChartConfigurationQuery(AggregateRequestDtoV2 request, JsonNode query, String indexName) { + String aggrQuery = query.get(Constants.JsonPaths.AGGREGATION_QUERY).asText(); + String rqMs = query.get(Constants.JsonPaths.REQUEST_QUERY_MAP).asText(); + String dateReferenceField = query.get(Constants.JsonPaths.DATE_REF_FIELD).asText(); + JsonNode requestQueryMaps = null; + ObjectNode objectNode = null; + ObjectMapper mapper = new ObjectMapper(); + Map esFilterMap = new HashMap<>(); + try { + requestQueryMaps = new ObjectMapper().readTree(rqMs); + request.setEsFilters(esFilterMap); + if(query.get(Constants.JsonPaths.MODULE).asText().equals(Constants.Modules.COMMON) && + !request.getModuleLevel().equals(Constants.Modules.HOME_REVENUE) && + !request.getModuleLevel().equals(Constants.Modules.HOME_SERVICES)) { + request.getFilters().put(Constants.Filters.MODULE, request.getModuleLevel()); + } + Iterator> filtersItr = request.getFilters().entrySet().iterator(); + while(filtersItr.hasNext()) { + Entry entry = filtersItr.next(); + if(!String.valueOf(entry.getValue()).equals(Constants.Filters.FILTER_ALL)) { + String esQueryKey = requestQueryMaps.get(entry.getKey()).asText(); + request.getEsFilters().put(esQueryKey, entry.getValue()); + } + } + ElasticSearchDictator dictator = elasticSearchDao.createSearchDictatorV2(request, indexName, "", dateReferenceField); + SearchRequest searchRequest = elasticSearchDao.buildElasticSearchQuery(dictator); + JsonNode querySegment = mapper.readTree(searchRequest.source().toString()); + objectNode = (ObjectNode) querySegment; + objectNode.put(Constants.JsonPaths.AGGS, mapper.readTree(aggrQuery).get(Constants.JsonPaths.AGGS)); + } catch (Exception ex) { + logger.error("Encountered an Exception while parsing the JSON : " + ex.getMessage()); + } + return objectNode; + + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceTemplate.java b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceTemplate.java new file mode 100644 index 000000000..9ec75a129 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/QueryServiceTemplate.java @@ -0,0 +1,95 @@ +package com.tarento.analytics.service.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.tomcat.util.codec.binary.Base64; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; +import org.springframework.stereotype.Component; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static javax.servlet.http.HttpServletRequest.BASIC_AUTH; +import static org.apache.commons.codec.CharEncoding.US_ASCII; +import static org.springframework.http.HttpHeaders.AUTHORIZATION; + +@Component +public class QueryServiceTemplate { + public static final Logger LOGGER = LoggerFactory.getLogger(QueryServiceTemplate.class); + + @Value("${egov.services.esindexer.host.name}") + private String indexServiceHost; + @Value("${egov.services.esindexer.host.search}") + private String indexServiceHostSearch; + @Value("${services.esindexer.host}") + private String dssindexServiceHost; + @Value("${egov-es-username}") + private String userName; + @Value("${egov-es-password}") + private String password; + + + @Autowired + private RestTemplate restTemplate; + + + public JsonNode search(String index, String searchQuery) throws IOException { + + //TODO remove the check for dssindexServiceHost + String url =(/*index.contains("dss") ? dssindexServiceHost :*/ indexServiceHost) + index + indexServiceHostSearch; + HttpHeaders headers = /*index.contains("dss") ? new HttpHeaders() :*/ getHttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + LOGGER.info("Index Name : " + index); + LOGGER.info("Searching ES for Query: " + searchQuery); + HttpEntity requestEntity = new HttpEntity<>(searchQuery, headers); + String reqBody = requestEntity.getBody(); + JsonNode responseNode = null; + + try { + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, Object.class); + responseNode = new ObjectMapper().convertValue(response.getBody(), JsonNode.class); + LOGGER.info("RestTemplate response :- "+responseNode); + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + LOGGER.error("client error while searching ES : " + e.getMessage()); + } + return responseNode; + } + + private HttpHeaders getHttpHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.add(AUTHORIZATION, getBase64Value(userName, password)); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + headers.setContentType(MediaType.APPLICATION_JSON); + + List mediaTypes = new ArrayList<>(); + mediaTypes.add(MediaType.APPLICATION_JSON); + headers.setAccept(mediaTypes); + return headers; + } + + /** + * Helper Method to create the Base64Value for headers + * + * @param userName + * @param password + * @return + */ + private String getBase64Value(String userName, String password) { + String authString = String.format("%s:%s", userName, password); + byte[] encodedAuthString = Base64.encodeBase64(authString.getBytes(Charset.forName(US_ASCII))); + return String.format(BASIC_AUTH, new String(encodedAuthString)); + } + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticProperties.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticProperties.java new file mode 100644 index 000000000..c27e7b79a --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticProperties.java @@ -0,0 +1,73 @@ +package com.tarento.analytics.utils; + +public interface ElasticProperties { + + public interface Property { + final String TRANSACTION_TRANSDATE = "TransDate"; + final String TRANSACTION_STOREID = "StoreId"; + final String TRANSACTION_NETAMOUNT = "NetAmt"; + final String TARGET_TARGETDATE = "targetDate"; + final String TARGET_STORECODE = "storeCode"; + final String TARGET_VERSION = "version"; + final String TARGET_SALESVALUE = "salesValue"; + final String TARGET_AVGVALUE = "avgValue"; + final String TARGET_RECEIPTVALUE = "receiptsValue"; + final String ITEM_DETAILS_CATEGORY_ID = "itemDetails.categoryId"; + final String ITEM_DETAILS_PRICE = "itemDetails.price" ; + final String AVERAGE_RATING = "AvgRating"; + final String SORT_ORDER_DESCENDING = "desc"; + final String SORT_ORDER_ASCENDING = "asc"; + final String COUNTS = "counts"; + final String COUNTS_RATING = "CountsRating"; + final String FEEDBACK_VALUE_RESPONSE = "feedback_value"; + } + + public interface SuccessMessages { + final String STORE_USER = "User and Store successfully mapped" ; + final String STORE_TIMING = "Store Times entered successfully" ; + } + + public interface Query { + final String NESTED = "NESTED"; + final String MATCH_CONDITION = "MATCH"; + final String RANGE_CONDITION = "RANGE"; + final String AGGREGATION_CONDITION = "AGGREGATION"; + final String TRANSACTION_DATE_FIELD = "transDate"; + final String FEEDBACK_DATE_TIME = "serverDateTime"; + final String COUNT_STORE_CODE = "storeId"; + final String COUNT_RATING_VALUE = "value"; + final String COUNT_REASON_KEYWORD = "reasons.keyword"; + final String COUNT_GENDER_KEYWORD = "gender.keyword"; + final String COUNT_AGEGROUP_KEYWORD = "ageGroup.keyword"; + final String SUM = "SUM"; + final String VALUE_COUNT="value_count"; + final String AVG = "AVG"; + final String CUMMULATIVE_SUM="cumulative_sum"; + final String FIELD = "FIELD"; + final String COUNT = "COUNT"; + final String BUCKETS_PATH = "BUCKETS_PATH"; + final String DATE_HISTOGRAM = "DATE_HISTOGRAM"; + final String EXTENDED_BOUNDS = "EXTENDED_BOUNDS"; + final String PATH ="PATH"; + final String MIN = "MIN"; + final String MAX = "MAX"; + final String INTERVAL = "INTERVAL"; + final String HOUR ="HOUR"; + final String DAY ="DAY"; + final String DAY_OF_WEEK ="dayOfWeek"; + final String ASC="ASC"; + final String DESC ="DESC"; + + final String MINUTE ="MINUTE"; + final String MONTH ="MONTH"; + final String TERM ="TERMS"; + final String SIZE="SIZE"; + final String ORDER="ORDER"; + final Integer TOP_CSAT_STORE_COUNT = 5; + final String LABEL="label"; + + + + } + +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticSearchClient.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticSearchClient.java new file mode 100644 index 000000000..f5d3ef2ba --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ElasticSearchClient.java @@ -0,0 +1,48 @@ +package com.tarento.analytics.utils; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + + +@Component +public class ElasticSearchClient { + private final String hostName; + private final Integer port; + private final static String HTTP = "http"; + private final Integer topPerformanceCount; + private final String transactionIndex; + private final String transactionType; + private final String targetIndex; + private final String targetType; + + public ElasticSearchClient(@Value("${services.esindexer.host.name}") String hostName, + @Value("${services.esindexer.host.port}") Integer port, + @Value("${top.performance.store.count}") Integer topPerformanceCount, + @Value("${es.transaction.index.name}") String transactionIndex, + @Value("${es.transaction.document.type}") String transactionType, + @Value("${es.target.index.name}") String targetIndex, + @Value("${es.target.document.type}") String targetType){ + + this.hostName = hostName; + this.port = port; + this.topPerformanceCount = topPerformanceCount; + this.transactionIndex = transactionIndex; + this.transactionType = transactionType; + this.targetIndex = targetIndex; + this.targetType = targetType; + + } + + public RestHighLevelClient getClient() { + return new RestHighLevelClient( + RestClient.builder(new HttpHost(hostName, port,HTTP))); + + } + + + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JSONObjectUtil.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JSONObjectUtil.java new file mode 100644 index 000000000..6a7e33e2c --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JSONObjectUtil.java @@ -0,0 +1,41 @@ +package com.tarento.analytics.utils; +import org.springframework.beans.factory.annotation.Autowired; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; + +public class JSONObjectUtil { + @Autowired + public ObjectMapper mapper; + @Autowired + public Gson gson; + + + /** + * @return + */ + public static String getJsonString(ObjectMapper objectMapper,Object object) throws JsonProcessingException { + if(objectMapper != null){ + return objectMapper.writeValueAsString(object); + } + return null; + } + + public ObjectMapper getMapper() { + return mapper; + } + + public void setObjectMapper(ObjectMapper objectMapper){ + mapper=objectMapper; + } + + public Gson getGson() { + return gson; + } + + public void setGson(Gson gsonn) + { + gson = gsonn; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JsonKey.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JsonKey.java new file mode 100644 index 000000000..d60097bc7 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/JsonKey.java @@ -0,0 +1,20 @@ +/** + * + */ +package com.tarento.analytics.utils; + +/** + * @author Satish + * + */ +public class JsonKey { + + public static final String STATUS_CODE = "statusCode"; + public static final String STATUS = "statusInfo"; + public static final String STATUS_MESSAGE = "statusMessage"; + public static final String ERROR_MESSAGE = "errorMessage"; + + public static final String RESPONSE_DATA = "responseData"; + + +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/PathRoutes.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/PathRoutes.java new file mode 100644 index 000000000..7a78c5a0f --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/PathRoutes.java @@ -0,0 +1,41 @@ +package com.tarento.analytics.utils; + +public interface PathRoutes { + + public interface DashboardApi { + final String DASHBOARD_ROOT_PATH = "/dashboard"; + + final String TEST_PATH = "/test"; + + final String GET_CHART = "/getChart"; + final String GET_CHART_V2 = "/getChartV2"; + final String GET_DASHBOARD_CONFIG = "/getDashboardConfig"; + final String GET_HOME_CONFIG = "/getHomeConfig"; + final String GET_ALL_VISUALIZATIONS = "/getAllVisualizations"; + final String ADD_NEW_DASHBOARD = "/addNewDashboard"; + final String MAP_DASHBOARD_VISUALIZATIOn = "/mapVisualizationToDashboard"; + final String MAP_VISUALIZATION_ROLE = "/mapVisualizationToRole"; + final String GET_HEADER_DATA = "/getDashboardHeader"; + final String GET_FEEDBACK_MESSAGE="/getPulseFeedbackMessage"; + + } + + public interface MetadataApi { + final String METADATA_ROOT_PATH = "/meta"; + + final String GET_CATEGORIES = "/getCategories"; + final String GET_SUB_CATEGORIES = "/getSubCategories"; + final String GET_ITEMS = "/getItems"; + final String GET_MASTERS = "/getMasters"; + final String FLUSH_MASTERS = "/flushMasters"; + final String GET_PULSE_RATING_CONFIG="/getPulseRatingConfig"; + final String GET_RATING_CONFIG = "/getRatingConfiguration"; + final String GET_RATING_CONFIG_ENCODE = "/getRatingConfig"; + final String GET_CONFIG_VERSION = "/getConfigVersion"; + final String PULSE_VERIFY_ORG_PIN = "/verifyOrgs"; + final String PUT_ORG_INDEX ="/putIndex"; + final String PUT_QUERY ="/createQuery"; + + + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseCode.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseCode.java new file mode 100644 index 000000000..ac4dc3962 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseCode.java @@ -0,0 +1,87 @@ +package com.tarento.analytics.utils; + +import com.tarento.analytics.constant.Constants; + +/** + * + * @author Abhishek + * + */ +public enum ResponseCode { + UnAuthorised(Constants.UNAUTHORIZED_ID, Constants.UNAUTHORIZED), Success( + Constants.SUCCESS_ID, Constants.SUCCESS),FAILURE( + Constants.FAILURE_ID, Constants.PROCESS_FAIL); + /** + * error code contains int value + */ + private int errorCode; + /** + * errorMessage contains proper error message. + */ + private String errorMessage; + + + + /** + * @param errorCode + * @param errorMessage + */ + private ResponseCode(int errorCode, String errorMessage) { + this.errorCode = errorCode; + this.errorMessage = errorMessage; + } + + /** + * + * @param errorCode + * @return + */ + public String getMessage(int errorCode) { + return ""; + } + + /** + * @return + */ + public int getErrorCode() { + return errorCode; + } + + /** + * @param errorCode + */ + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + /** + * @return + */ + public String getErrorMessage() { + return errorMessage; + } + + /** + * @param errorMessage + */ + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + /** + * This method will provide status message based on code + * + * @param code + * @return String + */ + public static String getResponseMessage(int code) { + String value = ""; + ResponseCode responseCodes[] = ResponseCode.values(); + for (ResponseCode actionState : responseCodes) { + if (actionState.getErrorCode() == code) { + value = actionState.getErrorMessage(); + } + } + return value; + } +} diff --git a/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseGenerator.java b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseGenerator.java new file mode 100644 index 000000000..ed7a89845 --- /dev/null +++ b/dashboard-analytics/src/main/java/com/tarento/analytics/utils/ResponseGenerator.java @@ -0,0 +1,86 @@ +package com.tarento.analytics.utils; + + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class ResponseGenerator { + + + private static ObjectMapper objectMapper = new ObjectMapper(); + + public static String failureResponse() throws JsonProcessingException{ + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + return JSONObjectUtil.getJsonString(objectMapper,response); + } + + + public static String failureResponse(String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE,message); + actualResponse.putPOJO(JsonKey.STATUS,response); + + return JSONObjectUtil.getJsonString(objectMapper,actualResponse); + } + + public static String failureResponse(String errorCode, String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(errorCode, message); + response.put(JsonKey.STATUS_MESSAGE, + message); + response.put(JsonKey.ERROR_MESSAGE,message); + actualResponse.putPOJO(JsonKey.STATUS,response); + + return JSONObjectUtil.getJsonString(objectMapper,actualResponse); + } + + public static String successResponse(Object obj) throws JsonProcessingException { + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, ResponseCode.Success.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, ""); + actualResponse.putPOJO(JsonKey.STATUS,response); + if (obj != null) { + actualResponse.putPOJO(JsonKey.RESPONSE_DATA, obj); + } + + return JSONObjectUtil.getJsonString(objectMapper,actualResponse); + } + + /** + * this method will crate success response and send to controller. + * + * @return ObjectNode object. + */ + public static String successResponse(String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.Success.getErrorMessage()); + //response.put(JsonKey.SUCCESS_MESSAGE, message); + actualResponse.putPOJO(JsonKey.STATUS,response); + response.put(JsonKey.ERROR_MESSAGE, ""); + actualResponse.putPOJO(JsonKey.STATUS,response); + if (message != null) { + actualResponse.putPOJO(JsonKey.RESPONSE_DATA, message); + } + + return JSONObjectUtil.getJsonString(objectMapper,actualResponse); + } +} diff --git a/dashboard-analytics/src/main/resources/application.properties b/dashboard-analytics/src/main/resources/application.properties new file mode 100644 index 000000000..4fc633602 --- /dev/null +++ b/dashboard-analytics/src/main/resources/application.properties @@ -0,0 +1,55 @@ +server.port=8082 + +spring.datasource.url=jdbc:mysql://localhost:3306/analytics_db +spring.datasource.username=root +spring.datasource.password=tiger +spring.datasource.driver-class-name=com.mysql.jdbc.Driver +spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration, org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration + +#----------------------------- ELASTIC SEARCH CONFIGURATIONS ------------------------------# +services.esindexer.host=http://es.rain.idc.tarento.com:9200/ +services.esindexer.host.name=es.rain.idc.tarento.com +services.esindexer.alternate.host=https://egov-micro-dev.egovernments.org/elasticsearch/ +services.esindexer.alternate.host.name=https://egov-micro-dev.egovernments.org/elasticsearch/ +services.esindexer.alternate.host.port=9200 +services.esindexer.host.port=9200 +services.esindexer.username=deploy +services.esindexer.password=Deploy123 +#services.esindexer.host={{ es_host | b64decode }} +#services.esindexer.host.name={{ es_host_name | b64decode }} +#services.esindexer.host.port=9200 +#services.esindexer.username={{ es_user | b64decode }} +#services.esindexer.password={{ es_pwd | b64decode }} +es.index.name=_transaction +es.document.type=transaction +es.transaction.index.name=_transaction +es.transaction.document.type=transaction +es.target.index.name=store_target +es.target.document.type=storeTarget +es.index.name.storeTarget=store_target +es.index.name.feedback=_app_feedback +es.index.name.storeFeedback=rain_feedback +es.document.type.storeTarget=storeTarget +es.document.type.feedback=appFeedback +es.document.type.storeFeedback=rainFeedback + +#---------------------------------Microservice details------------------------------------# +services.user.host.name=localhost +services.user.host.port=8081 +services.user.host.url=/user + +#----------------------------- MISCELLANEOUS CONFIGURATION ----------------------------------# +top.performance.store.count=5 + + +#spring.datasource.url={{ datasource_url | b64decode }} +#spring.datasource.username={{ datasource_user | b64decode }} +#spring.datasource.password={{ datasource_pwd | b64decode }} +#spring.datasource.driver-class-name=com.mysql.jdbc.Driver + + +egov.services.esindexer.host.name=http://104.211.240.72:9200/ +egov.services.esindexer.host.search=/_search + +egov-es-username=egov-admin +egov-es-password=TUSYns9mEcRPy77n \ No newline at end of file diff --git a/dashboard-analytics/src/main/resources/application.properties.j2 b/dashboard-analytics/src/main/resources/application.properties.j2 new file mode 100644 index 000000000..2b63c7415 --- /dev/null +++ b/dashboard-analytics/src/main/resources/application.properties.j2 @@ -0,0 +1,75 @@ +server.port=8089 + + +spring.datasource.url={{ datasource_url | b64decode }} +spring.datasource.username={{ datasource_user | b64decode }} +spring.datasource.password={{ datasource_pwd | b64decode }} +spring.datasource.driver-class-name=com.mysql.jdbc.Driver + +#----------------------------- ELASTIC SEARCH CONFIGURATIONS ------------------------------# +services.esindexer.host={{ es_host | b64decode }} +services.esindexer.host.name={{ es_host_name | b64decode }} +services.esindexer.host.port=9200 +services.esindexer.username={{ es_user | b64decode }} +services.esindexer.password={{ es_pwd | b64decode }} +es.index.name=_transaction +es.document.type=transaction +es.transaction.index.name=_transaction +es.transaction.document.type=transaction +es.target.index.name=store_target +es.target.document.type=storeTarget +es.index.name.storeTarget=store_target +es.index.name.feedback=_app_feedback +es.index.name.storeFeedback=rain_feedback +es.document.type.storeTarget=storeTarget +es.document.type.feedback=appFeedback +es.document.type.storeFeedback=rainFeedback + +#---------------------------------Microservice details------------------------------------# +services.user.host.name=localhost +services.user.host.port=8081 +services.user.host.url=/user + +#----------------------------- MISCELLANEOUS CONFIGURATION ----------------------------------# +top.performance.store.count=5 + +#------------------------------ KAFKA CONFIGURATIONS ------------------------------# + +# KAFKA SERVER CONFIGURATIONS +kafka.config.bootstrap_server_config=localhost:9092 + +spring.kafka.consumer.value-deserializer=com.tarento.analytics.consumer.HashMapDeserializer +spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer +spring.kafka.consumer.group-id=rain-transaction-chain +spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer +spring.kafka.producer.value-serializer=org.springframework.kafka.support.serializer.JsonSerializer + +# KAFKA CONSUMER CONFIGURATIONS +kafka.consumer.config.auto_commit=true +kafka.consumer.config.auto_commit_interval=100 +kafka.consumer.config.session_timeout=15000 +kafka.consumer.config.group_id=rain-transaction-chain +kafka.consumer.config.auto_offset_reset=earliest + +# KAFKA PRODUCER CONFIGURATIONS +kafka.producer.config.retries_config=0 +kafka.producer.config.batch_size_config=100000 +kafka.producer.config.linger_ms_config=100 +kafka.producer.config.buffer_memory_config=66554432 + +#--------------------------- KAFKA TOPICS TO CONSUME ---------------------------# + +kafka.topics.notify.newmessage.name=NewContentMessage +kafka.topics.notify.newmessage.key=content + +kafka.topics.transaction.pushdata.name=RainTransactionData +kafka.topics.transaction.pushdata.key=transactiondata + +kafka.topics.target.pushdata.name=NewTargetData +kafka.topics.target.pushdata.key=targetdata + +kafka.topics.feedback.pushdata.name=NewFeedbackData +kafka.topics.feedback.pushdata.key=feedbackdata + +kafka.topics.rating.pushdata.name=RainRatingData +kafka.topics.rating.pushdata.key=ratingdata \ No newline at end of file diff --git a/dashboard-analytics/src/main/resources/schema/ChartApiConfig.json b/dashboard-analytics/src/main/resources/schema/ChartApiConfig.json new file mode 100644 index 000000000..23d2532e6 --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/ChartApiConfig.json @@ -0,0 +1,725 @@ +{ + "_comment": "Starts Common Charts, applied across module", + "totalApplication": { + "chartName": "Total Application", + "queries": [ + { + "module": "PT", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"Total Application\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "dateRefField": "Data.@timestamp" + }, + { + "module": "TL", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"Total Application\":{\"value_count\":{\"field\":\"Data.tradelicense.tenantid.keyword\"}}}}", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tradelicense.tenantid\" \r\n}", + "dateRefField": "Data.tradelicense.applicationdate" + }, + { + "module": "PGR", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"Total Application\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "dateRefField": "Data.dateOfComplaint" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Total Application" + ], + "insight": { + }, + "_comment": " totalApplication is the chartId" + }, + "closedApplication": { + "chartName": "Closed Application", + "queries": [ + { + "module": "PT", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"Closed Application\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}}}}}}}" + }, + { + "module": "TL", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"Closed Application\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}}}}}}}" + }, + { + "module": "PGR", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"Closed Application\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}}}}}}}" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Closed Application" + ], + "insight": { + }, + "_comment": " totalApplication is the chartId" + }, + "citizenRegistered": { + "chartName": "Citizen Registered", + "queries": [ + { + "module": "PGR", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"Citizen Registered\":{\"terms\":{\"field\":\"Data.citizen.uuid.keyword\"},\"aggs\":{\"type_count\":{\"cardinality\":{\"field\":\"Data.citizen.uuid.keyword\"}}}}}}", + "requestQueryMap": "{\"district\" : \"Data.tenantData.city.districtCode\", \"tenantId\" : \"Data.tenantId\", \"status\" : \"Data.status\"}", + "dateRefField": "Data.dateOfComplaint" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Citizen Registered" + ], + "insight": { + }, + "_comment": " totalApplication is the chartId" + }, + "totalApplicationDeptWise": { + "chartName": "Total Application: Department Wise", + "queries": [ + { + "module": "PT", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"Property Tax\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}" + }, + { + "module": "TL", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"Trade Licence\":{\"value_count\":{\"field\":\"Data.tradelicense.tenantid.keyword\"}}}}" + }, + { + "module": "PGR", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"PGR\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Property Tax", + "Trade Licence", + "PGR" + ], + "insight": { + }, + "_comment": " totalApplication is the chartId" + }, + "totalApplication&ClosedApplication": { + "chartName": "Total Application & Closed Application", + "queries": [ + { + "module": "PT", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"Total Application\":{\"date_histogram\":{\"field\":\"Data.@timestamp\",\"interval\":\"week\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.propertyId.keyword\"}}}},\"Closed Application\":{\"date_histogram\":{\"field\":\"Data.@timestamp\",\"interval\":\"week\"},\"aggs\":{\"Applications Closed\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}}}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.propertyId.keyword\"}}}}}}}}" + }, + { + "module": "TL", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\":{\"Total Application\":{\"date_histogram\":{\"field\":\"Data.tradelicense.applicationDate\",\"interval\":\"week\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tradelicense.applicationNumber.keyword\"}}}},\"Closed Application\":{\"date_histogram\":{\"field\":\"Data.tradelicense.@timestamp\",\"interval\":\"week\"},\"aggs\":{\"Applications Closed\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}},\"resolved\":{\"match\":{\"Data.status.keyword\":\"resolved\"}}}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tradelicense.applicationNumber.keyword\"}}}}}}}}" + }, + { + "module": "PGR", + "dateRefField": "Data.@timestamp", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tenantId\" \r\n}", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"Total Application\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"week\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}},\"Closed Application\":{\"date_histogram\":{\"field\":\"Data.dateOfComplaint\",\"interval\":\"week\"},\"aggs\":{\"Applications Closed\":{\"filters\":{\"filters\":{\"closed\":{\"match\":{\"Data.status.keyword\":\"closed\"}}}},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}}}" + } + ], + "chartType": "line", + "valueType": "number", + "action": "", + "drillChart": "none", + "documentType": "_doc", + "aggregationPaths": [ + "Total Application", + "Closed Application" + ], + "insight": { + }, + "_comment": " " + }, + "cumulativeCollection": { + "chartName": "Total Cumulative Collection Weekly", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\r\n \"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \n\"tenantId\" : \"domainObject.tenantId\"}", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"2019\":{\"date_histogram\":{\"field\":\"dataObject.Bill.billDetails.receiptDate\",\"interval\":\"week\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}}}}" + } + ], + "chartType": "line", + "valueType": "amount", + "action": "", + "drillChart": "none", + "documentType": "_doc", + "aggregationPaths": [ + "2019" + ], + "insight": { + }, + "_comment": " " + }, + "targetCollection": { + "chartName": "Target Collection", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\", \"tenantId\" : \"tenantIdForMunicipalCorporation.keyword\"}", + "dateRefField": "Timestamp", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Target Collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}" + } + ], + "chartType": "metric", + "valueType": "Amount", + "action": "", + "drillChart": "none", + "documentType": "_doc", + "aggregationPaths": [ + "Target Collection" + ], + "insight": { + }, + "_comment": " " + }, + "targetAchieved": { + "chartName": "Target Achieved", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\r\n \"module\" : \"businessService.keyword\", \n\"tenantId\" : \"tenantIdForMunicipalCorporation\"}", + "dateRefField": "Timestamp", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Actual collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\r\n \"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \n\"tenantId\" : \"domainObject.tenantId\"}", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}}" + } + + ], + "chartType": "metric", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "aggregationPaths": [ + "Total Collection", + "Actual collection" + ], + "insight": { + }, + "_comment": " " + }, + "totalCollection": { + "chartName": "Total Collection", + "queries": [ + { + "module": "COMMON", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}}", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\"}", + "dateRefField": "dataObject.Bill.billDetails.receiptDate" + } + ], + "chartType": "metric", + "valueType": "Amount", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "aggregationPaths": [ + "Total Collection" + ], + "insight": { + }, + "_comment": " " + }, + "totalCollectionDeptWise": { + "chartName": "Total Collection Department Wise", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\"}", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"Deptment\":{\"terms\":{\"field\":\"dataObject.Bill.billDetails.businessService.keyword\"},\"aggs\":{\"total\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}}}}" + } + ], + "chartType": "pie", + "valueType": "Amount", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "aggregationPaths": [ + "Deptment" + ], + "insight": { + }, + "_comment": " " + }, + "topPerformingUlbs": { + "chartName": "Top 3 Performing ULBs", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\r\n \"module\" : \"businessService.keyword\", \n\"tenantId\" : \"tenantIdForMunicipalCorporation\"}", + "dateRefField": "Timestamp", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Total Collection\":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}},\"Target Collection\":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\r\n \"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \n\"tenantId\" : \"domainObject.tenantId\"}", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"Total Collection\":{\"terms\":{\"field\":\"domainObject.tenantId.keyword\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}},\"Target Collection\":{\"terms\":{\"field\":\"domainObject.tenantId.keyword\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + } + ], + "chartType": "perform", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "plotLabel": "Target Achieved", + "order": "desc", + "limit": 3, + "aggregationPaths": [ + "Total Collection","Target Collection" + ], + "insight": { + }, + "_comment": " Top Performing Ulbs for target achieved" + }, + "bottomPerformingUlbs": { + "chartName": "Bottom 3 Performing ULBs", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"tenantIdForMunicipalCorporation\" }", + "dateRefField": "Timestamp", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"Total Collection\":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}},\"Target Collection\":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"domainObject.tenantId\" }", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"Total Collection\":{\"terms\":{\"field\":\"domainObject.tenantId.keyword\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}},\"Target Collection\":{\"terms\":{\"field\":\"domainObject.tenantId.keyword\"},\"aggs\":{\"Sum\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + } + ], + "chartType": "perform", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "plotLabel": "Target Achieved", + "order": "asc", + "limit": 3, + "aggregationPaths": [ + "Total Collection", "Target Collection" + ], + "insight": { + }, + "_comment": " Bottom Performing Ulbs for target achieved" + }, + "topPerformingUlbsCompletionRate": { + "chartName": "Top 3 Performing ULBs", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantId\" , \"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"Closed Application\":{\"filter\":{\"term\":{\"Data.status.keyword\":\"resolved\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}},\"Total Application\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}" + }, + { + "module": "PT", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantId\" , \"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"Closed Application\":{\"filter\":{\"term\":{\"Data.status.keyword\":\"resolved\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}},\"Total Application\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}" + } + ], + "chartType": "perform", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "plotLabel": "Completion Rate", + "order": "desc", + "limit": 3, + "aggregationPaths": [ + "Closed Application", + "Total Application" + ], + "insight": { + }, + "_comment": " Top Performing Ulbs for Completion rate" + }, + "bottomPerformingUlbsCompletionRate": { + "chartName": "Bottom 3 Performing ULBs", + "queries": [ + { + "module": "PGR", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantId\" , \"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp", + "indexName": "pgrindex-v1", + "aggrQuery": "{\"aggs\":{\"Closed Application\":{\"filter\":{\"term\":{\"Data.status.keyword\":\"resolved\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}},\"Total Application\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}" + }, + { + "module": "PT", + "requestQueryMap": "{\"module\" : \"businessService.keyword\",\"tenantId\" : \"Data.tenantId\" ,\"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"Closed Application\":{\"filter\":{\"term\":{\"Data.status.keyword\":\"resolved\"}},\"aggs\":{\"count\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}},\"Total Application\":{\"terms\":{\"field\":\"Data.tenantId.keyword\"},\"aggs\":{\"tenant_count\":{\"value_count\":{\"field\":\"Data.tenantId.keyword\"}}}}}}" + } + ], + "chartType": "perform", + "valueType": "percentage", + "drillChart": "none", + "documentType": "_doc", + "action": "percentage", + "plotLabel": "Completion Rate", + "order": "asc", + "limit": 3, + "aggregationPaths": [ + "Closed Application", + "Total Application" + ], + "insight": { + }, + "_comment": " Bottom Performing Ulbs for Completion rate" + }, + "_comment": "Starts PT specific Charts", + "collectionByUsageType": { + "chartName": "Collection by Usage Type", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"Usage Type\":{\"terms\":{\"field\":\"domainObject.propertyDetails.units.usageCategoryMajor.keyword\"},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}}}}}}" + } + ], + "chartType": "pie", + "valueType": "amount", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Usage Type" + ], + "insight": { + }, + "_comment": " collection/amount per usage type" + }, + "propertiesByUsageType": { + "chartName": "Properties by Usage Type", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\"}", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"Usage Type\":{\"terms\":{\"field\":\"domainObject.propertyDetails.units.usageCategoryMajor.keyword\"},\"aggs\":{\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}" + } + ], + "chartType": "pie", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Usage Type" + ], + "insight": { + }, + "_comment": " properties having assessmentNumber per usage type" + }, + "propertiesAssessed": { + "chartName": "Total Properties Accessed", + "queries": [ + { + "module": "PT", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"Assessed Properties\":{\"value_count\":{\"field\":\"Data.propertyDetails.assessmentNumber.keyword\"}}}}", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\" ,\"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Assessed Properties" + ], + "insight": { + }, + "_comment": " totol properties having assessmentNumber " + }, + "cumulativePropertiesAssessed": { + "chartName": "Total Cumulative Properties Accessed Weekly", + "queries": [ + { + "module": "PT", + "indexName": "ptindex-v1", + "aggrQuery": "{\"aggs\":{\"2019\":{\"date_histogram\":{\"field\":\"Data.propertyDetails.assessmentDate\",\"interval\":\"week\"},\"aggs\":{\"Count\":{\"value_count\":{\"field\":\"Data.propertyDetails.assessmentNumber.keyword\"}}}}}}", + "requestQueryMap": "{\"tenantId\" : \"Data.tenantId\" ,\"district\" : \"Data.tenantData.city.districtCode\"}", + "dateRefField": "Data.@timestamp" + } + ], + "chartType": "line", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "2019" + ], + "insight": { + }, + "_comment": " totol properties having assessmentNumber " + }, + "activeUlbs": { + "chartName": "Total Active ULBs", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"Active ULBs\":{\"terms\":{\"field\":\"dataObject.tenantId.keyword\"},\"aggs\":{\"type_count\":{\"cardinality\":{\"field\":\"dataObject.tenantId.keyword\"}}}}}}" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "Active ULBs" + ], + "insight": { + }, + "_comment": " total ULBs count" + }, + "demandCollectionIndexBoundaryRevenue": { + "chartName": "Demand & Collection Index Boundary", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"domainObject.tenantId.keyword\"},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}" + }, + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"businessService.keyword\", \"tenantId\" : \"tenantIdForMunicipalCorporation.keyword\"}", + "dateRefField": "Timestamp", + "indexName": "dss-target_v1", + "aggrQuery": "{\"aggs\":{\"ULBs \":{\"terms\":{\"field\":\"tenantIdForMunicipalCorporation.keyword\"},\"aggs\":{\"Target Collection\":{\"sum\":{\"field\":\"budgetProposedForMunicipalCorporation\"}}}}}}" + } + ], + "filterKeys": [ + "tenantId" + ], + "chartType": "table", + "valueType": "number", + "drillChart": "boundaryDrillDown", + "drillFields": [ + "Ward", + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Boundary", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Assessed Properties", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Assessed Properties": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "boundaryDrillDown": { + "kind": "drillDown", + "chartName": "Demand & Collection Index Boundary", + "queries": [ + { + "fieldMapping": [ + { + "Ward": "domainObject.ward.children.code.keyword" + } + ], + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"Ward \":{\"terms\":{\"field\":\"domainObject.ward.children.code.keyword\"},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}" + } + ], + "chartType": "table", + "valueType": "number", + "drillChart": "none", + "documentType": "_doc", + "action": "", + "plotLabel": "Ward", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Assessed Properties", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Assessed Properties": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "demandCollectionIndexUsageRevenue": { + "chartName": "Demand & Collection Index Usage Type", + "queries": [ + { + "module": "COMMON", + "requestQueryMap": "{\"module\" : \"dataObject.Bill.billDetails.businessService.keyword\", \"tenantId\" : \"dataObject.tenantId\", \"district\" : \"dataObject.tenantData.cityDistrictCode\" }", + "dateRefField": "dataObject.@timestamp", + "indexName": "dss-collection_v1", + "aggrQuery": "{\"aggs\":{\"UsageType \":{\"terms\":{\"field\":\"domainObject.propertyDetails.units.usageCategoryMajor.keyword\"},\"aggs\":{\"Total Collection\":{\"sum\":{\"field\":\"dataObject.Bill.billDetails.amountPaid\"}},\"Transactions\":{\"value_count\":{\"field\":\"dataObject.transactionId.keyword\"}},\"Assessed Properties\":{\"value_count\":{\"field\":\"domainObject.propertyDetails.assessmentNumber.keyword\"}}}}}}" + } + ], + "filterKeys": [ + "tenantId" + ], + "chartType": "table", + "valueType": "number", + "drillChart": "", + "drillFields": [ + "Ward", + "" + ], + "documentType": "_doc", + "action": "", + "plotLabel": "Usage Type", + "aggregationPaths": [ + "Total Collection", + "Transactions", + "Assessed Properties", + "Target Collection" + ], + "pathDataTypeMapping": [ + { + "Total Collection": "amount" + }, + { + "Transactions": "number" + }, + { + "Assessed Properties": "number" + }, + { + "Target Collection": "amount" + } + ], + "insight": { + }, + "_comment": "" + }, + "_comment": "Starts TL specific Charts", + "licenseIssued": { + "chartName": "Total Trade License Issued", + "queries": [ + { + "module": "TL", + "indexName": "tlindex-v1", + "aggrQuery": "{\"aggs\": {\"License Issued\": {\"value_count\": {\"field\": \"Data.tradelicense.licensenumber.keyword\"}}}}", + "requestQueryMap": "{\r\n \"district\" : \"Data.tenantData.city.districtCode\", \r\n\"tenantId\" : \"Data.tradelicense.tenantid\" \r\n, \r\n\"businessService\" : \"dataObject.Bill.billDetails.businessServices\" \r\n}", + "dateRefField": "Data.tradelicense.applicationdate" + } + ], + "chartType": "metric", + "valueType": "number", + "action": "", + "documentType": "_doc", + "drillChart": "none", + "aggregationPaths": [ + "License Issued" + ], + "insight": { + }, + "_comment": " licenseIssued is the Chart ID" + } +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/resources/schema/RoleDashboardConfig.json b/dashboard-analytics/src/main/resources/schema/RoleDashboardConfig.json new file mode 100644 index 000000000..3e3e966eb --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/RoleDashboardConfig.json @@ -0,0 +1,560 @@ +{ + "_comment": "TODO: denormalize the role & visualisations", + "roles" : [ + { + "roleId": 6, + "roleName" : "R1", + "isSuper" : "", + "orgId": "", + "dashboards": [ + { + "name": "My Dashboard", + "id": "home", + "isActive": "", + "visualizations": [ + { + "row": 1, + "name": "Revenue", + "vizArray": [ + { + "id": 11, + "name": "", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "charts": [ + { + "id": "totalCollection", + "name": "Total Collection", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetCollection", + "name": "Target Collection", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetAchieved", + "name": "Target Achieved", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 12, + "name": "Total Cumulative Collection", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "cumulativeCollection", + "name": "Weekly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 2, + "name": "Revenue", + "vizArray": [ + { + "id": 21, + "name": "Top Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "topPerformingUlbs", + "name": "Top Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 22, + "name": "Bottom Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbs", + "name": "Bottom Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 23, + "name": "Total Cumulative Collection: Department Wise", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "totalCollectionDeptWise", + "name": "Total Cumulative Collection: Department Wise", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 3, + "name": "Service", + "vizArray": [ + { + "id": 31, + "name": "", + "dimensions": { + "height": 450, + "width": 5 + }, + "vizType": "metric-collection", + "charts": [ + { + "id": "totalApplication", + "name": "Total Application", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "closedApplication", + "name": "Closed Application", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "", + "name": "SLA Achieved", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "citizenRegistered", + "name": "Citizen Registered", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 32, + "name": "Total Application & Closed Application", + "dimensions": { + "height": 450, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "totalApplication&ClosedApplication", + "name": "", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 4, + "name": "Service", + "vizArray": [ + { + "id": 41, + "name": "Top Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "topPerformingUlbsCompletionRate", + "name": "Top Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 42, + "name": "Bottom Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbsCompletionRate", + "name": "Bottom Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 43, + "name": "Total Applications: Department Wise", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "totalApplicationDeptWise", + "name": "Total Cumulative Collection: Department Wise", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + } + ] + }, + { + "name": "Property Tax", + "id": "propertyTax", + "isActive":"", + "visualizations": [ + { + "row": 1, + "name": "Revenue", + "vizArray": [ + { + "id": 11, + "name": "", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "charts": [ + { + "id": "totalCollection", + "name": "Total Collection", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetCollection", + "name": "Target Collection", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "targetAchieved", + "name": "Target Achieved", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 12, + "name": "Total Cumulative Collection", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "cumulativeCollection", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 2, + "name": "Revenue", + "vizArray": [ + { + "id": 21, + "name": "Top 3 Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "topPerformingUlbs", + "name": "Top 3 Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 22, + "name": "Bottom 3 Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbs", + "name": "Bottom 3 Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 23, + "name": "Collection by Usage type", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "collectionByUsageType", + "name": "Collection by Usage type", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 3, + "name": "Revenue", + "vizArray": [ + { + "id": 31, + "name": "", + "dimensions": { + "height": 350, + "width": 12 + }, + "vizType": "chart", + "charts": [ + { + "id": "demandCollectionIndexBoundaryRevenue", + "name": "Demand & Collection Index", + "code": "", + "chartType": "table", + "filter": "", + "headers": [] + }, + { + "id": "demandCollectionIndexUsageRevenue", + "name": "Demand & Collection Index", + "code": "", + "chartType": "table", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 4, + "name": "Service", + "vizArray": [ + { + "id": 41, + "name": "", + "dimensions": { + "height": 350, + "width": 5 + }, + "vizType": "metric-collection", + "charts": [ + { + "id": "propertiesAssessed", + "name": "Total Properties Assessed", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "propertiesAssessed", + "name": "Total Assessments", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + }, + { + "id": "activeUlbs", + "name": "Total Active ULBs", + "code": "", + "chartType": "metric", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 42, + "name": "Total Properties Assessed", + "dimensions": { + "height": 350, + "width": 7 + }, + "vizType": "chart", + "charts": [ + { + "id": "cumulativePropertiesAssessed", + "name": "Monthly", + "code": "", + "chartType": "line", + "filter": "", + "headers": [] + } + ] + } + ] + }, + { + "row": 5, + "name": "Service", + "vizArray": [ + { + "id": 21, + "name": "Top 3 Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "topPerformingUlbsCompletionRate", + "name": "Top 3 Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 22, + "name": "Bottom 3 Performing ULBs", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "performing-metric", + "charts": [ + { + "id": "bottomPerformingUlbsCompletionRate", + "name": "Bottom 3 Performing ULBs", + "code": "", + "chartType": "bar", + "filter": "", + "headers": [] + } + ] + }, + { + "id": 23, + "name": "Properties by Usage type", + "dimensions": { + "height": 250, + "width": 4 + }, + "vizType": "chart", + "charts": [ + { + "id": "propertiesByUsageType", + "name": "Properties by Usage type", + "code": "", + "chartType": "pie", + "filter": "", + "headers": [] + } + ] + } + ] + } + ] + } + ] + } + ] +} diff --git a/dashboard-analytics/src/main/resources/schema/getchart.json b/dashboard-analytics/src/main/resources/schema/getchart.json new file mode 100644 index 000000000..9f9e5ef89 --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/getchart.json @@ -0,0 +1,43 @@ +{ + "$schema": "ain/getchart/schema1#", + "title": "getchart", + "description": "getchart api schema", + "type": "object", + "properties": { + "header": { + "type": "object", + "properties": { + "tenantId": { + "type": "string" + } + } + }, + "requestInfo": { + "type": "object", + "properties": { + "chartType":{ + "type":"string", + "enum":["bar","line","stackedbar","pie", "horizontalBar", "doughnut", "doubledonut","heat","radar"] + }, + "chartFormat":{ + "type":"string" + }, + "serviceApi":{ + "type":"string" + }, + "dates":{ + "type":"object", + "properties":{ + "startDate":{ + "type":"string" + }, + "endData":{ + "type":"string" + } + } + } + }, + "required": ["chartType","chartFormat", "serviceApi","dates"] + } + } +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/resources/schema/query.json b/dashboard-analytics/src/main/resources/schema/query.json new file mode 100644 index 000000000..32aeae976 --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/query.json @@ -0,0 +1,20 @@ + + +{ + "y_axis": { + "aggregation": { + "sum": { + "field": "netAmt" + }, + "label": "revenue" + } + }, + "x_axis": { + "aggregation": { + "term": { + "field": "storeId" + }, + "label":"stores" + } + } +} \ No newline at end of file diff --git a/dashboard-analytics/src/main/resources/schema/query.yaml b/dashboard-analytics/src/main/resources/schema/query.yaml new file mode 100644 index 000000000..84f02cc63 --- /dev/null +++ b/dashboard-analytics/src/main/resources/schema/query.yaml @@ -0,0 +1,9 @@ +------! Bar char + +{ + + + + +} + \ No newline at end of file diff --git a/dashboard-ingest/pom.xml b/dashboard-ingest/pom.xml new file mode 100644 index 000000000..61947ce89 --- /dev/null +++ b/dashboard-ingest/pom.xml @@ -0,0 +1,195 @@ + + 4.0.0 + + com.ingestpipeline + ingestpipeline + 0.0.1-SNAPSHOT + jar + + ingestpipeline + + + org.springframework.boot + spring-boot-starter-parent + 2.1.8.RELEASE + + + + 1.8 + UTF-8 + UTF-8 + 1.8 + 3.0.1 + + + + + org.springframework.boot + spring-boot-starter-web + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + + + org.springframework.kafka + spring-kafka + + + + + com.google.code.gson + gson + 2.8.0 + + + com.bazaarvoice.jolt + jolt-core + 0.1.1 + + + com.bazaarvoice.jolt + json-utils + 0.1.1 + + + javax.ws.rs + jsr311-api + 1.1.1 + + + org.apache.commons + commons-lang3 + 3.4 + + + commons-io + commons-io + 2.6 + + + org.everit.json + org.everit.json.schema + 1.3.0 + + + org.elasticsearch + elasticsearch + 6.6.0 + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.6.0 + + + commons-fileupload + commons-fileupload + 1.4 + + + org.apache.poi + poi + 4.1.0 + + + org.apache.poi + poi-ooxml + 4.1.0 + + + + commons-beanutils + commons-beanutils + 1.8.3 + + + net.sf.ezmorph + ezmorph + 1.0.6 + + + commons-collections + commons-collections + 3.2.1 + + + commons-lang + commons-lang + 2.6 + + + org.json + json + 20160810 + + + + + com.fasterxml.jackson.core + jackson-databind + 2.9.8 + + + + org.springframework.boot + spring-boot-starter-data-jpa + 2.1.4.RELEASE + + + com.h2database + h2 + runtime + 1.4.199 + + + org.apache.commons + commons-lang3 + 3.7 + + + + com.github.wnameless + json-flattener + 0.2.2 + + + + + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.springframework + spring-jdbc + + + org.json + json + 20190722 + + + + com.google.guava + guava + 20.0 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/AppStartupRunner.java b/dashboard-ingest/src/main/java/com/ingestpipeline/AppStartupRunner.java new file mode 100644 index 000000000..df03e69ff --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/AppStartupRunner.java @@ -0,0 +1,42 @@ +package com.ingestpipeline; + +import com.ingestpipeline.controller.RestApiController; +import com.ingestpipeline.model.CollectionDomainConfig; +import com.ingestpipeline.util.ConfigLoader; +import com.ingestpipeline.util.ReadUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.stereotype.Component; + +/** + * The App Startup Runner runs on the start of the application as it implements Application Runner + * This will be responsible to load the configurations which are necessary for the Enrichment in the Data Pipeline + * Resources and Domain level configurations are fetched here. + * @author Pritha + * + */ +@Component +public class AppStartupRunner implements ApplicationRunner { + + private static Logger logger = LoggerFactory.getLogger(AppStartupRunner.class); + + @Autowired + ConfigLoader configLoader; + @Autowired + CollectionDomainConfig collectionDomainConfig; + + @Autowired ReadUtil readutil; + + @Autowired RestApiController restApiController; + + @SuppressWarnings("static-access") + @Override + public void run(ApplicationArguments args) throws Exception { + logger.info("On Boot starts loading: config resources "); + configLoader.loadResources(); + collectionDomainConfig.loadCollectionDomains(); + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/IngestApp.java b/dashboard-ingest/src/main/java/com/ingestpipeline/IngestApp.java new file mode 100644 index 000000000..8f5f90b19 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/IngestApp.java @@ -0,0 +1,37 @@ +package com.ingestpipeline; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; + +import com.ingestpipeline.util.Constants; + + +@SpringBootApplication(scanBasePackages={"com.ingestpipeline"})// same as @Configuration @EnableAutoConfiguration @ComponentScan combined +public class IngestApp { + + public static void main(String[] args) { + SpringApplication.run(IngestApp.class, args); + } + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + + @Bean + public WebMvcConfigurer corsConfigurer() { + return new WebMvcConfigurerAdapter() { + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/**").allowedMethods(Constants.ALLOWED_METHODS_GET,Constants.ALLOWED_METHODS_POST + ).allowedOrigins("*") + .allowedHeaders("*"); + } + }; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/config/ElasticSearchConfiguration.java b/dashboard-ingest/src/main/java/com/ingestpipeline/config/ElasticSearchConfiguration.java new file mode 100644 index 000000000..6f4a1c744 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/config/ElasticSearchConfiguration.java @@ -0,0 +1,66 @@ +package com.ingestpipeline.config; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.AbstractFactoryBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.ingestpipeline.service.ElasticService; +import com.ingestpipeline.service.IESService; + +@Configuration +public class ElasticSearchConfiguration extends AbstractFactoryBean { + + private static final Logger logger = LoggerFactory.getLogger(ElasticSearchConfiguration.class); + @Value("${spring.data.elasticsearch.cluster-nodes}") + private String clusterNodes; + @Value("${spring.data.elasticsearch.cluster-name}") + private String clusterName; + private RestHighLevelClient restHighLevelClient; + @Value("${services.esindexer.host.name}") + private String esIndexerHostName; + @Value("${services.esindexer.host.port}") + private int esIndexerHostPort; + + @Override + public void destroy() { + try { + if (restHighLevelClient != null) { + restHighLevelClient.close(); + } + } catch (final Exception e) { + logger.error("Error closing ElasticSearch client: ", e); + } + } + + @Override + public Class getObjectType() { + return RestHighLevelClient.class; + } + + @Override + public boolean isSingleton() { + return false; + } + + @Override + public RestHighLevelClient createInstance() { + return buildClient(); + } + + private RestHighLevelClient buildClient() { + try { + restHighLevelClient = new RestHighLevelClient( + RestClient.builder(new HttpHost(esIndexerHostName, esIndexerHostPort, "http"))); + } catch (Exception e) { + logger.error(e.getMessage()); + } + return restHighLevelClient; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ConsumerConfigurations.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ConsumerConfigurations.java new file mode 100644 index 000000000..cf59516b9 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ConsumerConfigurations.java @@ -0,0 +1,127 @@ +package com.ingestpipeline.consumer; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.support.serializer.JsonDeserializer; + +/** + * The configurations for the Kafka Consumers are fed from this Config Class + * @author Darshan Nagesh + * + */ +@PropertySource(value= {"classpath:application.properties"}) +@Configuration +@EnableKafka +public class ConsumerConfigurations { + + @Autowired + org.springframework.core.env.Environment env; + + @Value("${kafka.config.bootstrap_server_config}") + private String serverConfig; + + @Value("${kafka.consumer.config.auto_commit}") + private Boolean enableAutoCommit; + + @Value("${kafka.consumer.config.auto_commit_interval}") + private String autoCommitInterval; + + @Value("${kafka.consumer.config.session_timeout}") + private String sessionTimeout; + + @Value("${kafka.consumer.config.group_id}") + private String groupId; + + @Value("${kafka.consumer.config.auto_offset_reset}") + private String autoOffsetReset; + + @Value("${spring.kafka.consumer.value-deserializer}") + private String valueDeserializer; + + @Value("${spring.kafka.consumer.key-deserializer}") + private String keyDeserializer; + + + public ConsumerFactory kafkaConsumerFactory() { + JsonDeserializer deserializer = new JsonDeserializer<>(Map.class); + //deserializer.setRemoveTypeHeaders(false); + deserializer.addTrustedPackages("*"); + deserializer.setUseTypeMapperForKey(true); + + Map props = new HashMap<>(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "task-group-notify"); + return new DefaultKafkaConsumerFactory<>(props, new StringDeserializer(), deserializer); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory incomingKafkaListenerContainerFactory() { + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(kafkaConsumerFactory()); + return factory; + } + + + + /*@Bean + KafkaListenerContainerFactory> kafkaListenerContainerFactory() { + System.out.println("kafkaListenerContainerFactory"); + final ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory()); + factory.setConcurrency(3); + factory.getContainerProperties().setPollTimeout(3000); + return factory; + } + + @Bean + public ConsumerFactory consumerFactory() { + System.out.println("consumerFactory"); + return new DefaultKafkaConsumerFactory<>(consumerConfigs()); + } + + @Bean + public Map consumerConfigs() { + // TODO - Load configs from env vars + final Map propsMap = new HashMap<>(); + propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, serverConfig); + propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit); + propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval); + propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout); + propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); + propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, HashMapDeserializer.class); +*//* propsMap.put(JsonDeserializer., "*"); +*//* return propsMap; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory incomingKafkaListenerContainerFactory() { + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(taskConsumerFactory()); + return factory; + } + + public ConsumerFactory taskConsumerFactory() { + Map props = new HashMap<>(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "task-group-notify"); + return new DefaultKafkaConsumerFactory<>(props, new StringDeserializer(), new JsonDeserializer<>(IncomingData.class)); + } +*/ + /* + * @Bean public TransactionPersistConsumer listener() { return new TransactionPersistConsumer(); } + */ +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/EnrichmentConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/EnrichmentConsumer.java new file mode 100644 index 000000000..a7293402e --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/EnrichmentConsumer.java @@ -0,0 +1,45 @@ +package com.ingestpipeline.consumer; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.stereotype.Service; + +import com.ingestpipeline.producer.IngestProducer; +import com.ingestpipeline.service.EnrichmentService; +import com.ingestpipeline.util.Constants; + +@Service +public class EnrichmentConsumer implements KafkaConsumer { + + public static final Logger LOGGER = LoggerFactory.getLogger(EnrichmentConsumer.class); + private static final String ERROR_INTENT = "DataError"; + private static final String INTENT = "enrichment" ; + + @Autowired + private EnrichmentService enrichmentService; + + @Autowired + private IngestProducer ingestProducer; + + @KafkaListener(id = INTENT, groupId = INTENT, topics = { Constants.KafkaTopics.TRANSFORMED_DATA}, containerFactory = Constants.BeanContainerFactory.INCOMING_KAFKA_LISTENER) + public void processMessage(final Map incomingData, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic) { + LOGGER.info("##KafkaMessageAlert## : key:" + topic + ":" + "value:" + incomingData); + + try { + boolean denormStatus = enrichmentService.enrichData(incomingData); + if(!denormStatus) { + ingestProducer.pushToPipeline(incomingData, ERROR_INTENT, ERROR_INTENT); + } + } catch (final Exception e) { + LOGGER.error("Exception Encountered while processing the received message : " + e.getMessage()); + } + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/HashMapDeserializer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/HashMapDeserializer.java new file mode 100644 index 000000000..76941d4f3 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/HashMapDeserializer.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.consumer; + +import java.util.HashMap; + +import org.springframework.kafka.support.serializer.JsonDeserializer; + +@SuppressWarnings("rawtypes") +public class HashMapDeserializer extends JsonDeserializer { + + public HashMapDeserializer() { + super(HashMap.class); + } + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/KafkaConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/KafkaConsumer.java new file mode 100644 index 000000000..69374842c --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/KafkaConsumer.java @@ -0,0 +1,15 @@ +package com.ingestpipeline.consumer; + +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; + +import com.ingestpipeline.model.IncomingData; + +import java.util.Map; + +public interface KafkaConsumer { + + public void processMessage(final Map incomingData, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/TransformConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/TransformConsumer.java new file mode 100644 index 000000000..e4d918b06 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/TransformConsumer.java @@ -0,0 +1,48 @@ +package com.ingestpipeline.consumer; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.stereotype.Service; + +import com.ingestpipeline.producer.IngestProducer; +import com.ingestpipeline.service.TransformService; +import com.ingestpipeline.util.ApplicationProperties; +import com.ingestpipeline.util.Constants; + +@Service +public class TransformConsumer implements KafkaConsumer { + + public static final Logger LOGGER = LoggerFactory.getLogger(TransformConsumer.class); + public static final String INTENT = "transform"; + @Autowired + private TransformService transformService; + + @Autowired + private IngestProducer ingestProducer; + + @Autowired + private ApplicationProperties applicationProperties; + + @Override + @KafkaListener(id = INTENT, groupId = INTENT, topics = { Constants.KafkaTopics.VALID_DATA }, containerFactory = Constants.BeanContainerFactory.INCOMING_KAFKA_LISTENER) + public void processMessage(Map incomingData, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic) { + LOGGER.info("##KafkaMessageAlert## : key:" + topic + ":" + "value:" + incomingData); + try { + boolean isTransformed = transformService.transformData(incomingData); + if (isTransformed) { + ingestProducer.pushToPipeline(incomingData, applicationProperties.getTransactionTransformationTopic(), applicationProperties.getTransactionTransformationKey()); + } else { + ingestProducer.pushToPipeline(incomingData, Constants.KafkaTopics.ERROR_INTENT, Constants.KafkaTopics.ERROR_INTENT); + } + } catch (final Exception e) { + LOGGER.error("Exception Encountered while processing the received message : " + e.getMessage()); + } + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ValidatorConsumer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ValidatorConsumer.java new file mode 100644 index 000000000..cacf93822 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/consumer/ValidatorConsumer.java @@ -0,0 +1,64 @@ +package com.ingestpipeline.consumer; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.stereotype.Service; + +import com.ingestpipeline.producer.IngestProducer; +import com.ingestpipeline.service.ValidationService; +import com.ingestpipeline.util.ApplicationProperties; +import com.ingestpipeline.util.Constants; + +@Service +public class ValidatorConsumer implements KafkaConsumer { + + public static final Logger LOGGER = LoggerFactory.getLogger(ValidatorConsumer.class); + + public static final String INTENT = "validator"; + + @Autowired + private ValidationService validationService; + + @Autowired + private IngestProducer ingestProducer; + + @Autowired + private ApplicationProperties applicationProperties; + + @KafkaListener(id = INTENT, groupId = INTENT, topics = {Constants.KafkaTopics.INGEST_DATA} , containerFactory = Constants.BeanContainerFactory.INCOMING_KAFKA_LISTENER) + public void processMessage(Map consumerRecord, + @Header(KafkaHeaders.RECEIVED_TOPIC) final String topic) { + LOGGER.info("##KafkaMessageAlert## Message Received at Validator Consumer : key:" + topic + ":" + "value:" + consumerRecord); + try { + boolean isValid = validationService.validateData(consumerRecord); + String nextTopic = ""; + String nextKey = ""; + if (isValid) { + if(applicationProperties.getPipelineRules().get(Constants.PipelineRules.TRANSFORM_DATA)) { + nextTopic = applicationProperties.getTransactionValidationTopic(); + nextKey = applicationProperties.getTransactionValidationKey(); + } else if(applicationProperties.getPipelineRules().get(Constants.PipelineRules.ENRICH_DATA)) { + nextTopic = applicationProperties.getTransactionTransformationTopic(); + nextKey = applicationProperties.getTransactionTransformationKey(); + } + ingestProducer.pushToPipeline(consumerRecord, nextTopic, nextKey); + } else { + ingestProducer.pushToPipeline(consumerRecord, Constants.KafkaTopics.ERROR_INTENT, Constants.KafkaTopics.ERROR_INTENT); + } + LOGGER.info("Next Topic: " + nextTopic); + LOGGER.info("Next Key: " + nextKey); + + } catch (final Exception e) { + LOGGER.error("Exception Encountered while processing the received message : " + e.getMessage()); + } + + + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/controller/RestApiController.java b/dashboard-ingest/src/main/java/com/ingestpipeline/controller/RestApiController.java new file mode 100644 index 000000000..7ca9ee06d --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/controller/RestApiController.java @@ -0,0 +1,201 @@ +package com.ingestpipeline.controller; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.*; + +import com.ingestpipeline.model.TargetData; +import com.ingestpipeline.repository.TargetDataDao; +import org.apache.poi.EncryptedDocumentException; +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.kafka.config.KafkaListenerEndpointRegistry; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import com.bazaarvoice.jolt.modifier.DataType.MAP; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.ingestpipeline.model.IncomingData; +import com.ingestpipeline.service.IngestService; +import com.ingestpipeline.util.Constants; +import com.ingestpipeline.util.JSONUtil; +import com.ingestpipeline.util.JSONUtils; +import com.ingestpipeline.util.ReadUtil; +import com.ingestpipeline.service.ElasticService; + +@RestController +@RequestMapping(Constants.Paths.ELASTIC_PUSH_CONTROLLER_PATH) +public class RestApiController { + + public static final Logger logger = LoggerFactory.getLogger(RestApiController.class); + + @Autowired + IngestService ingestService; + + @Autowired + TargetDataDao targetDataDao; + + @Autowired + private KafkaListenerEndpointRegistry endPointRegistry; + + @Autowired + private ElasticService elasticService; + + /** + * This API use to pause a active kafka consumer + * + * @param consumerId kafka consumer identifier + * @return + */ + @RequestMapping(value = "/pause/{consumerId}", method = RequestMethod.GET) + public Boolean pauseConsumer(@PathVariable final String consumerId) { + endPointRegistry.getListenerContainer(consumerId).pause(); + return Boolean.TRUE; + } + + /** + * This API is to resume a paused kafka consumer + * + * @param consumerId kafka consumer identifier + * @return + */ + @RequestMapping(value = "/resume/{consumerId}", method = RequestMethod.GET) + public Boolean resumeConsumer(@PathVariable final String consumerId) { + endPointRegistry.getListenerContainer(consumerId).resume(); + return Boolean.TRUE; + } + + /** + * This API receives the Transaction Details JSON Request and passes it on to + * the Service Layer for further process of persisting into elastic search + * database + * + * @param transaction + * @return + */ + @RequestMapping(value = Constants.Paths.SAVE, method = RequestMethod.POST) + public ResponseEntity save(@RequestBody IncomingData incomingData) { + logMyTime(); + Boolean status = ingestService.ingestToPipeline(incomingData); + if (status) { + return new ResponseEntity(HttpStatus.CREATED); + } + return new ResponseEntity(HttpStatus.SERVICE_UNAVAILABLE); + } + + /** + * This API use to provide response for external data upload + * + * @param get file + * @author Rahul + * @throws Exception + */ + @RequestMapping(value = Constants.Paths.Targets, method = RequestMethod.GET) + public String getTargets() throws Exception { + List datas = (List) targetDataDao.findAll(); + String response = JSONUtil.getJsonString(new ObjectMapper(), datas); + return response; + } + + /** + * This API use to import external data upload Request + * + * @param upload file + * @author Rahul + * @throws Exception + */ + @RequestMapping(value = Constants.Paths.UPLOAD, method = RequestMethod.POST) + public ResponseEntity handleFileUpload(@RequestParam("file") MultipartFile file) throws Exception { + Boolean status = ingestService.ingestToPipeline(getWrapper(file)); + if (status) { + return new ResponseEntity(HttpStatus.CREATED); + } + return new ResponseEntity(HttpStatus.SERVICE_UNAVAILABLE); + } + + public IncomingData getWrapper(MultipartFile file) throws Exception { + JSONArray jsonArray = new JSONArray(); + jsonArray = ReadUtil.getFiletoDirectory(file); + IncomingData incomingData = new IncomingData(); + incomingData.setDataContext("target"); + incomingData.setDataContextVersion("v1"); + Iterator itr = jsonArray.iterator(); + List list = new ArrayList(); + while (itr.hasNext()) { + JSONObject obj = (JSONObject) itr.next(); + Map mapJ = new Gson().fromJson(obj.toString(), Map.class); + list.add(mapJ); + } + incomingData.setDataObject(list); + return incomingData; + } + + /** + * This API to post documents from ES index + * + * @param post documents + * @author Rahul + * @throws Exception + */ + @RequestMapping(value = Constants.Paths.ES_INDEX, method = RequestMethod.POST) + public ResponseEntity migrateIndex(@PathVariable String indexName) throws Exception { + String index = null, queryString = null, dataContext = null; + if (indexName.equals(Constants.ES_INDEX_COLLECTION)) { + index = Constants.ES_INDEX_COLLECTION; + queryString = elasticService.getSearchQueryCollection(); + dataContext = "collection"; + } else if (indexName.equals(Constants.ES_INDEX_BILLING)) { + index = Constants.ES_INDEX_BILLING; + queryString = elasticService.getSearchQueryBilling(); + dataContext = "billing"; + } else { + index = "notDefinedIndex"; + queryString = "noquery"; + } + Boolean status = false; + if (!index.equals("notDefinedIndex")) { + Map> doc_Map = elasticService.searchIndex(index, queryString); + for (Map.Entry> entry : doc_Map.entrySet()) { + for (int i = 0; i < entry.getValue().size(); i++) { + status = ingestService + .ingestToPipeline(setIncomingData(dataContext, "v1", entry.getValue().get(i))); + } + } + } + if (status) { + return new ResponseEntity(HttpStatus.CREATED); + } else if (index.equals("notDefinedIndex")) { + return new ResponseEntity(HttpStatus.NOT_FOUND); + } + return new ResponseEntity(HttpStatus.SERVICE_UNAVAILABLE); + } + + private IncomingData setIncomingData(String index, String version, Object documentValue) { + IncomingData incomingData = new IncomingData(); + incomingData.setDataContext(index); + incomingData.setDataContextVersion(version); + incomingData.setDataObject(documentValue); + return incomingData; + } + + private void logMyTime() { + logger.info("System Time is : " + new Date()); + SimpleDateFormat sd = new SimpleDateFormat(Constants.DATE_FORMAT); + Date date = new Date(); + sd.setTimeZone(TimeZone.getTimeZone(Constants.INDIAN_TIMEZONE)); + logger.info("Time at timezone IST : " + sd.format(date)); + } +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/CollectionDomainConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/CollectionDomainConfig.java new file mode 100644 index 000000000..eeaed76fb --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/CollectionDomainConfig.java @@ -0,0 +1,83 @@ +package com.ingestpipeline.model; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.ingestpipeline.util.ConfigLoader; + +//@JsonIgnoreProperties(ignoreUnknown=true) +@Component("CollectionDomainConfig") +public class CollectionDomainConfig { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectionDomainConfig.class); + private static final String COLLECTION_DOMAIN_CONFIG = "CollectionDomainConfig.json"; + + + @Autowired + private ConfigLoader configLoader; + /** + * Holds domain name as key and it's index config detail as value. + */ + private Map domainIndexConfigMap = new HashMap<>(); + + public void putDomain(String domainName, DomainIndexConfig domainIndexConfig){ + domainIndexConfigMap.put(domainName, domainIndexConfig); + } + + public DomainIndexConfig getIndexConfig(String domainName){ + return domainIndexConfigMap.get(domainName); + } + + + /** + * loads once on application start up. + */ + public void loadCollectionDomains(){ + //TODO:- load the JSON mapping, read/prepare the n queries + String collectionConfigContent = configLoader.get(COLLECTION_DOMAIN_CONFIG); + LOGGER.info("collectionConfigContent json string = "+collectionConfigContent); + + try{ + + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = new ObjectMapper().readTree(collectionConfigContent); + ArrayNode domainConfigArr = (ArrayNode) root.path("domainConfig"); + + Iterator iterator = domainConfigArr.elements(); + while (iterator.hasNext()) { + DomainIndexConfig domainIndexConfig = mapper.readValue(iterator.next().toString(), DomainIndexConfig.class); + System.out.println("DomainIndexConfig id=" + domainIndexConfig.getBusinessType()); + domainIndexConfigMap.put(domainIndexConfig.getBusinessType(), domainIndexConfig); + + } + LOGGER.info("After loading, domainIndexConfigMap size = "+ domainIndexConfigMap.size()); + + } catch (Exception e){ + e.printStackTrace(); + LOGGER.error("on construction domain collection map: "+ e.getMessage()); + } + + } + + + +/* private List domainIndexConfig; + @JsonProperty(value="domainConfig") + public List getDomainIndexConfig() { + return domainIndexConfig; + } + + public void setDomainIndexConfig(List domainIndexConfig) { + this.domainIndexConfig = domainIndexConfig; + }*/ + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/CollectionRef.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/CollectionRef.java new file mode 100644 index 000000000..3410cb382 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/CollectionRef.java @@ -0,0 +1,38 @@ +package com.ingestpipeline.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class CollectionRef { + + + private String fieldName; + private String argument; + private String dataType; + + @JsonProperty(value="fieldName") + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + @JsonProperty(value="argument") + public String getArgument() { + return argument; + } + + public void setArgument(String argument) { + this.argument = argument; + } + + @JsonProperty(value="dataType") + public String getDataType() { + return dataType; + } + + public void setDataType(String dataType) { + this.dataType = dataType; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/Consumable.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Consumable.java new file mode 100644 index 000000000..7f34f9f95 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Consumable.java @@ -0,0 +1,58 @@ +package com.ingestpipeline.model; + + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"dataContext", +"topics" +}) +public class Consumable { + +@JsonProperty("dataContext") +private String dataContext; +@JsonProperty("topics") +private List topics = null; +@JsonIgnore +private Map additionalProperties = new HashMap(); + +@JsonProperty("dataContext") +public String getDataContext() { +return dataContext; +} + +@JsonProperty("dataContext") +public void setDataContext(String dataContext) { +this.dataContext = dataContext; +} + +@JsonProperty("topics") +public List getTopics() { +return topics; +} + +@JsonProperty("topics") +public void setTopics(List topics) { +this.topics = topics; +} + +@JsonAnyGetter +public Map getAdditionalProperties() { +return this.additionalProperties; +} + +@JsonAnySetter +public void setAdditionalProperty(String name, Object value) { +this.additionalProperties.put(name, value); +} + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/DomainIndexConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/DomainIndexConfig.java new file mode 100644 index 000000000..dc04626b4 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/DomainIndexConfig.java @@ -0,0 +1,71 @@ +package com.ingestpipeline.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; + +public class DomainIndexConfig { + + private String id; + private String businessType; + private String indexName; + private String documentType; + private String query; + + private List collectionRef = new ArrayList<>(); + + @JsonProperty(value="id") + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + @JsonProperty(value="businessType") + public String getBusinessType() { + return businessType; + } + + public void setBusinessType(String businessType) { + this.businessType = businessType; + } + + @JsonProperty(value="indexName") + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + @JsonProperty(value="documentType") + public String getDocumentType() { + return documentType; + } + + public void setDocumentType(String documentType) { + this.documentType = documentType; + } + + @JsonProperty(value="query") + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + @JsonProperty(value="collectionRef") + public List getCollectionRef() { + return collectionRef; + } + + public void setCollectionRef(List collectionRef) { + this.collectionRef = collectionRef; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingData.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingData.java new file mode 100644 index 000000000..ce02ff52f --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingData.java @@ -0,0 +1,35 @@ +package com.ingestpipeline.model; + +public class IncomingData { + + private String dataContext; + private String dataContextVersion; + private Object dataObject; + + public String getDataContext() { + return dataContext; + } + public void setDataContext(String dataContext) { + this.dataContext = dataContext; + } + public String getDataContextVersion() { + return dataContextVersion; + } + public void setDataContextVersion(String dataContextVersion) { + this.dataContextVersion = dataContextVersion; + } + public Object getDataObject() { + return dataObject; + } + public void setDataObject(Object dataObject) { + this.dataObject = dataObject; + } + @Override + public String toString() { + return "IncomingData [dataContext=" + dataContext + ", dataContextVersion=" + dataContextVersion + + ", dataObject=" + dataObject + "]"; + } + + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingDataConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingDataConfig.java new file mode 100644 index 000000000..f9ac55dce --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/IncomingDataConfig.java @@ -0,0 +1,45 @@ +package com.ingestpipeline.model; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"consumables" +}) +public class IncomingDataConfig { + +@JsonProperty("consumables") +private List consumables = null; +@JsonIgnore +private Map additionalProperties = new HashMap(); + +@JsonProperty("consumables") +public List getConsumables() { +return consumables; +} + +@JsonProperty("consumables") +public void setConsumables(List consumables) { +this.consumables = consumables; +} + +@JsonAnyGetter +public Map getAdditionalProperties() { +return this.additionalProperties; +} + +@JsonAnySetter +public void setAdditionalProperty(String name, Object value) { +this.additionalProperties.put(name, value); +} + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/KeyValuePair.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/KeyValuePair.java new file mode 100644 index 000000000..3556caf66 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/KeyValuePair.java @@ -0,0 +1,72 @@ +package com.ingestpipeline.model; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"name", +"value", +"intent" +}) +public class KeyValuePair { + +@JsonProperty("intent") +private String intent; +@JsonProperty("name") +private String name; +@JsonProperty("value") +private String value; +@JsonIgnore +private Map additionalProperties = new HashMap(); + +@JsonProperty("name") +public String getName() { +return name; +} + +@JsonProperty("name") +public void setName(String name) { +this.name = name; +} + +@JsonProperty("intent") +public String getIntent() { +return intent; +} + +@JsonProperty("intent") +public void setIntent(String intent) { +this.intent = intent; +} + + + +@JsonProperty("value") +public String getValue() { +return value; +} + +@JsonProperty("value") +public void setValue(String value) { +this.value = value; +} + +@JsonAnyGetter +public Map getAdditionalProperties() { +return this.additionalProperties; +} + +@JsonAnySetter +public void setAdditionalProperty(String name, Object value) { +this.additionalProperties.put(name, value); +} + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/Snippet.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Snippet.java new file mode 100644 index 000000000..69d1e509c --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/Snippet.java @@ -0,0 +1,8 @@ +package com.ingestpipeline.model; + +public class Snippet { + public static void main(String[] args) { + + } +} + diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetData.java b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetData.java new file mode 100644 index 000000000..168bc94cf --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/model/TargetData.java @@ -0,0 +1,148 @@ +package com.ingestpipeline.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.io.Serializable; +import java.math.BigDecimal; + +@Entity +@Table(name = "TargetData") +@JsonIgnoreProperties +public class TargetData implements Serializable { + + @Id /*@GeneratedValue(strategy=GenerationType.AUTO) */ + @Column(name = "id", nullable = false) + private int id; + + @Column(name = "snoForMunicipalCorporation", nullable = false) + private String snoForMunicipalCorporation; + + @Column(name = "tenantIdForMunicipalCorporation", nullable = false) + private String tenantIdForMunicipalCorporation; + + @Column(name = "ulbName", nullable = false) + private String ulbName; + + @Column(name = "actualCollectionForMunicipalCorporation", nullable = false) + private BigDecimal actualCollectionForMunicipalCorporation; + + @Column(name = "budgetProposedForMunicipalCorporation", nullable = false) + private BigDecimal budgetProposedForMunicipalCorporation; + + @Column(name = "actualCollectionBudgetedForMunicipalCorporation", nullable = false) + private BigDecimal actualCollectionBudgetedForMunicipalCorporation; + + @Column(name = "financialYear", nullable = false) + private String financialYear; + + @Column(name = "Timestamp", nullable = false) + private String Timestamp; + + @Column(name = "businessService", nullable = false) + private String businessService; + + public TargetData() { + super(); + // TODO Auto-generated constructor stub + } + + @JsonProperty("id") + public int getId() { + return id; + } + @JsonProperty("id") + public void setId(int id) { + this.id = id; + } + @JsonProperty("snoForMunicipalCorporation") + public String getSnoForMunicipalCorporation() { + return snoForMunicipalCorporation; + } + + public void setSnoForMunicipalCorporation(String snoForMunicipalCorporation) { + this.snoForMunicipalCorporation = snoForMunicipalCorporation; + } + @JsonProperty("tenantIdForMunicipalCorporation") + public String getTenantIdForMunicipalCorporation() { + return tenantIdForMunicipalCorporation; + } + + public void setTenantIdForMunicipalCorporation(String tenantIdForMunicipalCorporation) { + this.tenantIdForMunicipalCorporation = tenantIdForMunicipalCorporation; + } + @JsonProperty("ulbName") + public String getUlbName() { + return ulbName; + } + @JsonProperty("ulbName") + public void setUlbName(String ulbName) { + this.ulbName = ulbName; + } + @JsonProperty("actualCollectionForMunicipalCorporation") + public BigDecimal getActualCollectionForMunicipalCorporation() { + return actualCollectionForMunicipalCorporation; + } + @JsonProperty("actualCollectionForMunicipalCorporation") + public void setActualCollectionForMunicipalCorporation(BigDecimal actualCollectionForMunicipalCorporation) { + this.actualCollectionForMunicipalCorporation = actualCollectionForMunicipalCorporation; + } + @JsonProperty("budgetProposedForMunicipalCorporation") + public BigDecimal getBudgetProposedForMunicipalCorporation() { + return budgetProposedForMunicipalCorporation; + } + @JsonProperty("budgetProposedForMunicipalCorporation") + public void setBudgetProposedForMunicipalCorporation(BigDecimal budgetProposedForMunicipalCorporation) { + this.budgetProposedForMunicipalCorporation = budgetProposedForMunicipalCorporation; + } + @JsonProperty("actualCollectionBudgetedForMunicipalCorporation") + public BigDecimal getActualCollectionBudgetedForMunicipalCorporation() { + return actualCollectionBudgetedForMunicipalCorporation; + } + @JsonProperty("actualCollectionBudgetedForMunicipalCorporation") + public void setActualCollectionBudgetedForMunicipalCorporation(BigDecimal actualCollectionBudgetedForMunicipalCorporation) { + this.actualCollectionBudgetedForMunicipalCorporation = actualCollectionBudgetedForMunicipalCorporation; + } + @JsonProperty("financialYear") + public String getFinancialYear() { + return financialYear; + } + @JsonProperty("financialYear") + public void setFinancialYear(String financialYear) { + this.financialYear = financialYear; + } + @JsonProperty("Timestamp") + public String getTimestamp() { + return Timestamp; + } + @JsonProperty("Timestamp") + public void setTimestamp(String timestamp) { + Timestamp = timestamp; + } + @JsonProperty("businessService") + public String getBusinessService() { + return businessService; + } + @JsonProperty("businessService") + public void setBusinessService(String businessService) { + this.businessService = businessService; + } + + @Override + public String toString() { + return "TargetData [id=" + id + ", snoForMunicipalCorporation=" + snoForMunicipalCorporation + + ", tenantIdForMunicipalCorporation=" + tenantIdForMunicipalCorporation + ", ulbName=" + ulbName + + ", actualCollectionForMunicipalCorporation=" + actualCollectionForMunicipalCorporation + + ", budgetProposedForMunicipalCorporation=" + budgetProposedForMunicipalCorporation + + ", actualCollectionBudgetedForMunicipalCorporation=" + actualCollectionBudgetedForMunicipalCorporation + + ", financialYear=" + financialYear + ", Timestamp=" + Timestamp + ", businessService=" + + businessService + "]"; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/producer/HashMapSerializer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/HashMapSerializer.java new file mode 100644 index 000000000..b2840c3c3 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/HashMapSerializer.java @@ -0,0 +1,33 @@ +package com.ingestpipeline.producer; + +import java.util.Map; + +import org.apache.kafka.common.serialization.Serializer; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class HashMapSerializer implements Serializer { + @Override + public void close() { + // TODO Auto-generated method stub + } + + @Override + public void configure(Map arg0, boolean arg1) { + // TODO Auto-generated method stub + } + + @Override + public byte[] serialize(String topic, Map data) { + byte[] value = null; + ObjectMapper objectMapper = new ObjectMapper(); + try { + value = objectMapper.writeValueAsString(data).getBytes(); + } catch (JsonProcessingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return value; + } +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducer.java new file mode 100644 index 000000000..b370385bd --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducer.java @@ -0,0 +1,24 @@ +package com.ingestpipeline.producer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.stereotype.Service; + +import com.google.gson.Gson; + +@Service +public class IngestProducer { + + public static final Logger LOGGER = LoggerFactory.getLogger(IngestProducer.class); + + @Autowired + private KafkaTemplate kafkaTemplate; + + public void pushToPipeline(Object object, String topic, String key) { + LOGGER.info("Request before pushing to Kafka Queue : " + new Gson().toJson(object)); + LOGGER.info("Kafka Topic : " + topic + " Kafka Key : " + key); + kafkaTemplate.send(topic, key, object); + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducerConfig.java b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducerConfig.java new file mode 100644 index 000000000..f05efe868 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/IngestProducerConfig.java @@ -0,0 +1,67 @@ +package com.ingestpipeline.producer; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.support.serializer.JsonSerializer; + +@Configuration +@EnableKafka +public class IngestProducerConfig { + + @Value("${kafka.config.bootstrap_server_config}") + private String serverConfig; + + @Value("${kafka.producer.config.retries_config}") + private Integer retriesConfig; + + @Value("${kafka.producer.config.batch_size_config}") + private Integer batchSizeConfig; + + @Value("${kafka.producer.config.linger_ms_config}") + private Integer lingerMsConfig; + + @Value("${kafka.producer.config.buffer_memory_config}") + private Integer bufferMemoryConfig; + + @Bean + public ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, serverConfig); + configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); + return new DefaultKafkaProducerFactory<>(configProps); + } +/* @Bean + public ProducerFactory producerFactory() { + return new DefaultKafkaProducerFactory<>(producerConfigs()); + } + + @Bean + public Map producerConfigs() { + final Map props = new HashMap<>(); + + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, serverConfig); + props.put(ProducerConfig.RETRIES_CONFIG, retriesConfig); + props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSizeConfig); + props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMsConfig); + props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemoryConfig); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); + return props; + }*/ + + @Bean + public KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/producer/JavaSerializer.java b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/JavaSerializer.java new file mode 100644 index 000000000..b99d21395 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/producer/JavaSerializer.java @@ -0,0 +1,37 @@ +package com.ingestpipeline.producer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.Map; + +import org.apache.kafka.common.serialization.Serializer; + +public class JavaSerializer implements Serializer { + + @Override + public byte[] serialize(String topic, Object data) { + try { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + ObjectOutputStream objectStream = new ObjectOutputStream(byteStream); + objectStream.writeObject(data); + objectStream.flush(); + objectStream.close(); + return byteStream.toByteArray(); + } + catch (IOException e) { + throw new IllegalStateException("Can't serialize object: " + data, e); + } + } + + @Override + public void configure(Map configs, boolean isKey) { + + } + + @Override + public void close() { + + } + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/repository/ElasticSearchRepository.java b/dashboard-ingest/src/main/java/com/ingestpipeline/repository/ElasticSearchRepository.java new file mode 100644 index 000000000..0af60ece0 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/repository/ElasticSearchRepository.java @@ -0,0 +1,58 @@ +package com.ingestpipeline.repository; + +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RestTemplate; + +/** + * This Repository Class is used to perform the transactions of storing the data into the Elastic Search Repository + * @author Darshan Nagesh + * + */ +@Service +public class ElasticSearchRepository { + + public static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchRepository.class); + + private final RestTemplate restTemplate; + + public ElasticSearchRepository(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + } + + /** + * Based on the Transaction Index Data Obtained and the URL with Headers, this method will put the Data obtained on the + * Elastic Search Database and returns the response in the form of Positive or Negative outcome (True Or False) + * @param transactionIndex + * @param url + * @param headers + * @return + */ + public Boolean saveMyDataObject(Object object, String url, HttpHeaders headers) { + ResponseEntity map = null; + try { + map = restTemplate.exchange(url, HttpMethod.PUT, + new HttpEntity<>(object, headers), Map.class); + } catch (final HttpClientErrorException httpClientErrorException) { + LOGGER.error("Error : " + httpClientErrorException); + } catch (HttpServerErrorException httpServerErrorException) { + LOGGER.error("Error : " + httpServerErrorException); + } catch (Exception e) { + LOGGER.error("Error : " + e); + } + if (map != null && map.getStatusCode() != null && (map.getStatusCode() == HttpStatus.OK) || (map.getStatusCode() == HttpStatus.CREATED)) { + return true; + } + return false; + } + } diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/repository/TargetDataDao.java b/dashboard-ingest/src/main/java/com/ingestpipeline/repository/TargetDataDao.java new file mode 100644 index 000000000..50fea5867 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/repository/TargetDataDao.java @@ -0,0 +1,18 @@ +package com.ingestpipeline.repository; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.ingestpipeline.model.TargetData; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface TargetDataDao extends CrudRepository { + +/* public List findAll(); + + public void save(TargetData targetData);*/ +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/ElasticService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ElasticService.java new file mode 100644 index 000000000..0b117edea --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ElasticService.java @@ -0,0 +1,233 @@ +package com.ingestpipeline.service; + +import com.eclipsesource.json.JsonObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.ingestpipeline.model.TargetData; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.search.SearchHit; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; +import org.springframework.stereotype.Component; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import org.springframework.http.*; +import java.io.IOException; +import java.util.Date; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Component("elasticService") +public class ElasticService implements IESService { + + @Value("${egov.services.esindexer.host.name}") + private String indexServiceHost; + @Value("${egov.services.esindexer.host.search}") + private String indexServiceHostSearch; + + @Value("${es.host.schema}") + private String schema; + + @Value("${services.esindexer.host}") + private String indexerServiceHost; + @Value("${es.target.index.name}") + private String targetIndexName; + + @Value("${es.index.name}") + private String collectionIndexName; + + @Value("${es.index.searchQuery.collection}") + private String searchQueryCollection; + + @Value("${es.index.searchQuery.billing}") + private String searchQueryBilling; + + @Autowired + private RestTemplate restTemplate; + + public static final Logger LOGGER = LoggerFactory.getLogger(ElasticService.class); + + public String getSearchQueryCollection() { + return searchQueryCollection; + } + + public void setSearchQueryCollection(String searchQueryCollection) { + this.searchQueryCollection = searchQueryCollection; + } + + public String getSearchQueryBilling() { + return searchQueryBilling; + } + + public void setSearchQueryBilling(String searchQueryBilling) { + this.searchQueryBilling = searchQueryBilling; + } + + @Override + public JsonNode search(String index, ObjectNode searchQuery) throws IOException { + SearchRequest searchRequest = buildSearchRequest(index, searchQuery); + SearchResponse searchResponse = getClient(index, indexServiceHost, 9200, schema).search(searchRequest, + RequestOptions.DEFAULT); + + ArrayNode resultArray = JsonNodeFactory.instance.arrayNode(); + for (SearchHit hit : searchResponse.getHits()) { + JsonNode node = new ObjectMapper().readValue(hit.getSourceAsString(), JsonNode.class); + resultArray.add(node); + } + return resultArray; + } + + @Override + public Map search(String index, String searchQuery) throws Exception { + + String url = indexServiceHost + index + indexServiceHostSearch; + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + LOGGER.info("searching ES for query: " + searchQuery + " on " + index); + + HttpEntity requestEntity = new HttpEntity<>(searchQuery, headers); + + try { + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, requestEntity, Object.class); + Map responseNode = new ObjectMapper().convertValue(response.getBody(), Map.class); + Map hits = (Map)responseNode.get("hits"); + if((Integer)hits.get("total") >=1) + return (Map)((ArrayList)hits.get("hits")).get(0); + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + LOGGER.error("client error while searching ES : " + e.getMessage()); + + } + return null; + } + + @Override + public Boolean push(Map requestBody) throws Exception { + + String trId = ((Map) requestBody.get("dataObject")).get("transactionId").toString(); + String url = indexerServiceHost + collectionIndexName + "/_doc/" + trId; + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + LOGGER.info("Posting request to ES on " + collectionIndexName); + JsonNode request = new ObjectMapper().convertValue(requestBody, JsonNode.class); + + HttpEntity requestEntity = new HttpEntity<>(request.toString(), headers); + ArrayNode hitNodes = null; + + try { + ResponseEntity response = restTemplate.exchange(url, HttpMethod.PUT, requestEntity, Object.class); + LOGGER.info("Status code on pushing to collection index : " + response.getStatusCode()); + if (response.getStatusCode().value() == 201) + return Boolean.TRUE; + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + LOGGER.error("client error while pushing ES collection index : " + e.getMessage()); + + } + return Boolean.FALSE; + } + + @Override + public Boolean push(TargetData requestBody) throws Exception { + + Long currentDateTime = new Date().getTime(); + String url = indexerServiceHost + targetIndexName + "/_doc/" + requestBody.getId(); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + System.out.println("Posting request to ES on " + targetIndexName); + JsonNode request = new ObjectMapper().convertValue(requestBody, JsonNode.class); + + HttpEntity requestEntity = new HttpEntity<>(request.toString(), headers); + ArrayNode hitNodes = null; + + try { + ResponseEntity response = restTemplate.exchange(url, HttpMethod.PUT, requestEntity, Object.class); + System.out.println("Status code on pushing to target index : " + response.getStatusCode()); + if (response.getStatusCode().value() == 201) + return Boolean.TRUE; + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + System.out.println("client error while pushing ES target index : " + e.getMessage()); + + } + return Boolean.FALSE; + } + + @Override + public Map searchIndex(String index, String searchQuery) throws Exception { + String scrollUrl = indexServiceHost + index + indexServiceHostSearch + "?scroll=5m"; + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + LOGGER.info("searching ES for query: " + searchQuery + " on " + index); + + HttpEntity requestEntity = new HttpEntity<>(searchQuery, headers); + Map hits = new LinkedHashMap(); + String scroll_id = null; + String str = null; + String queryForScrollId = null; + String postByScrollId = null; + Map> hitsToMap = new LinkedHashMap(); + try { + ResponseEntity response = restTemplate.exchange(scrollUrl, HttpMethod.POST, requestEntity, + Object.class); + Map responseNode = new ObjectMapper().convertValue(response.getBody(), Map.class); + str = indexServiceHostSearch.replaceAll("[/]", ""); + scroll_id = (String) responseNode.get("_scroll_id"); + postByScrollId = indexServiceHost + str + "/" + "scroll"; + queryForScrollId = "{\"scroll\":\"5m\",\"scroll_id\":" + "\"" + scroll_id + "\"" + "}"; + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + LOGGER.error("client error while searching ES : " + e.getMessage()); + } + try { + requestEntity = new HttpEntity<>(queryForScrollId, headers); + ResponseEntity response = restTemplate.exchange(postByScrollId, HttpMethod.POST, requestEntity, + Object.class); + Map responseNode = new ObjectMapper().convertValue(response.getBody(), Map.class); + hits = (Map) responseNode.get("hits"); + if ((Integer) hits.get("total") >= 1) { + hitsToMap.put("hits", ((ArrayList) hits.get("hits"))); + return hitsToMap; + } + + } catch (HttpClientErrorException e) { + e.printStackTrace(); + LOGGER.error("client error while searching ES : " + e.getMessage()); + } + return hits; + } + + +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichTransform.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichTransform.java new file mode 100644 index 000000000..d9b5e96fa --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichTransform.java @@ -0,0 +1,51 @@ +package com.ingestpipeline.service; + +import com.bazaarvoice.jolt.Chainr; +import com.bazaarvoice.jolt.JsonUtils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +public class EnrichTransform { + + @Autowired + private TransformService transformService; + + private static final String SEPARATOR = "_"; + private static final String JSON_EXTENSION = ".json"; + private static final String OBJECTIVE = "transform"; + private static final String CONFIGROOT = "config/"; + private static final String VERSION = "v1"; + + + + /** + * Tranforms domain raw response from elastic search + * This transformation is specific to domain objects + * + * @param rawResponseNode + * @param businessService + * @return + */ + public Object transform (Map rawResponseNode, String businessService) { + + String sourceUrl = CONFIGROOT.concat(OBJECTIVE.concat(SEPARATOR).concat(businessService.toLowerCase()).concat(SEPARATOR).concat(VERSION).concat(JSON_EXTENSION)); + List chainrSpecJSON = JsonUtils.jsonToList(this.getClass().getClassLoader().getResourceAsStream(sourceUrl)); + Chainr chainr = Chainr.fromSpec( chainrSpecJSON ); + + Object indexData = rawResponseNode.keySet().contains("_source") ? ((Map)rawResponseNode.get("_source")).get("Data") : null; + Object transNode = indexData!= null ? chainr.transform(indexData) : null; + + return transNode; + + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentService.java new file mode 100644 index 000000000..c9ab3d9dc --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentService.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.service; + +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; + +import java.util.Map; + +@Service +public interface EnrichmentService { + + Boolean enrichData(Map incomingData); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentServiceImpl.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentServiceImpl.java new file mode 100644 index 000000000..2bf0003e0 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/EnrichmentServiceImpl.java @@ -0,0 +1,201 @@ +package com.ingestpipeline.service; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.hash.Hashing; +import com.ingestpipeline.model.CollectionDomainConfig; +import com.ingestpipeline.model.DomainIndexConfig; +import com.ingestpipeline.model.TargetData; +import com.ingestpipeline.repository.TargetDataDao; + +import com.ingestpipeline.util.JSONUtil; +import org.apache.tomcat.util.codec.binary.Base64; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; + +import com.ingestpipeline.repository.ElasticSearchRepository; +import com.ingestpipeline.util.Constants; + +/** + * This is a Service Implementation for all the actions which are with respect + * to Elastic Search + * + * @author Darshan Nagesh + * + */ +@Service(Constants.Qualifiers.ENRICHMENT_SERVICE) +public class EnrichmentServiceImpl implements EnrichmentService { + + public static final Logger LOGGER = LoggerFactory.getLogger(EnrichmentServiceImpl.class); + private static final String SEPARATOR = "_"; + private static final String JSON_EXTENSION = ".json"; + private static final String OBJECTIVE = "enrichment"; + private final String indexServiceHost; + private final String userName; + private final String password; + private final String elasticSearchIndexName; + private final String elasticSearchDocumentType; + + private static final String AUTHORIZATION = "Authorization"; + private static final String US_ASCII = "US-ASCII"; + private static final String BASIC_AUTH = "Basic %s"; + + private static final String BUSINESS_SERVICE = "businessService"; + private static final String DATA_OBJECT = "dataObject"; + + @Autowired + private ElasticSearchRepository elasticRepository; + + @Autowired + private CollectionDomainConfig collectionDomainConfig; + + @Autowired + private IESService elasticService; + + @Autowired + private TargetDataDao targetDataDao; + + @Autowired + private EnrichTransform enrichTransform; + + public EnrichmentServiceImpl(@Value("${services.esindexer.host}") String indexServiceHost, + @Value("${services.esindexer.username}") String userName, + @Value("${services.esindexer.password}") String password, + @Value("${es.index.name}") String elasticSearchIndexName, + @Value("${es.document.type}") String elasticSearchDocumentType) { + this.indexServiceHost = indexServiceHost; + this.userName = userName; + this.password = password; + this.elasticSearchIndexName = elasticSearchIndexName; + this.elasticSearchDocumentType = elasticSearchDocumentType; + } + + @Override + public Boolean enrichData(Map incomingData) { + if (incomingData.get("dataContext").toString().equalsIgnoreCase("target")) { + ArrayNode values = new ObjectMapper().convertValue(incomingData.get(Constants.DATA_OBJECT), ArrayNode.class); + //System.out.println("incomingData 1209 values " + values); + + values.forEach(val -> { + TargetData targetData = new ObjectMapper().convertValue(val, TargetData.class); + String hashId = targetData.getFinancialYear() + "-" + targetData.getBusinessService()+"-"+targetData.getUlbName(); + hashId = hashId.replaceAll("(\\s)+", ""); + String sha256hex = Hashing.sha256().hashString(hashId, StandardCharsets.UTF_8).toString(); + targetData.setId(sha256hex.hashCode()); + //targetData.setId(hashId); + try{ + elasticService.push(targetData); + }catch (Exception e ){ + e.printStackTrace(); + } + + }); + } else if(incomingData.get("dataContext").toString().equalsIgnoreCase("collection")){ + // prepare the query required based on incoming data businessType + ObjectNode incomingNode = new ObjectMapper().convertValue(incomingData.get(DATA_OBJECT), ObjectNode.class); + ObjectNode copyNode = incomingNode.deepCopy(); + String businessTypeVal = copyNode.findValue(BUSINESS_SERVICE).asText(); + + DomainIndexConfig indexConfig = collectionDomainConfig.getIndexConfig(businessTypeVal.toString()); + String indexName = indexConfig.getIndexName(); + String query = indexConfig.getQuery(); + + try { + ObjectNode queryNode = new ObjectMapper().readValue(query, ObjectNode.class); + + indexConfig.getCollectionRef().forEach(collectionRef -> { + StringBuffer argVal = new StringBuffer(); + String argument = collectionRef.getArgument(); + String fieldName = collectionRef.getFieldName(); + + JSONUtil.findValue(incomingNode, fieldName, argVal); // argument value is search with fieldname and stored in argVal + + if (businessTypeVal.toString().equalsIgnoreCase("PT") && argument.equalsIgnoreCase("Data.propertyId")) { + String [] array = argVal.toString().split(":"); + argVal = new StringBuffer(); + argVal.append(array[0]); + + } else if (businessTypeVal.toString().equalsIgnoreCase("PT") && argument.equalsIgnoreCase("Data.propertyDetails.assessmentNumber")) { + String [] array = argVal.toString().split(":"); + argVal = new StringBuffer(); + argVal.append(array[1]); + } + + JSONUtil.replaceFieldValue(queryNode, argument, argVal.toString()); // construct query: argument node value replaced by argVal + LOGGER.info("Query node "+ queryNode); + + + }); + // Hit Elastic search: pull record for the constructed query + Map domainNode = elasticService.search(indexName, queryNode.toString()); + LOGGER.info("Fetched record from ES of a businessType "+ businessTypeVal + ": " + domainNode); + + //transform the request(s) using schema(s), and reform the incoming object + Object transDomainResponse = enrichTransform.transform(domainNode, businessTypeVal.toString()); + + incomingData.put("domainObject", transDomainResponse); + //System.out.println("collection result: "+new ObjectMapper().convertValue(incomingData, JsonNode.class)); + + //Push that transformed incoming object to ES - Collection new index. + elasticService.push(incomingData); + + }catch (Exception e) { + e.printStackTrace(); + LOGGER.error("Pre-processing - Fetching record from ES for " + businessTypeVal + "failed: " + e.getMessage()); + } + + } + + return Boolean.TRUE; + } + + private Boolean pushToElasticSearchIndex(Object object) { + Long currentDateTime = new Date().getTime(); + String url = String.format("%s%s/%s/%s", this.indexServiceHost, elasticSearchIndexName, + elasticSearchDocumentType, currentDateTime); + HttpHeaders headers = getHttpHeaders(); + LOGGER.info("Data Object to be added to ES : " + object); + LOGGER.info("URL to invoke : " + url); + elasticRepository.saveMyDataObject(object, url, headers); + return Boolean.TRUE; + } + + /** + * A helper method to create the headers for Rest Connection with UserName and + * Password + * + * @return HttpHeaders + */ + private HttpHeaders getHttpHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.add(AUTHORIZATION, getBase64Value(userName, password)); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + headers.setContentType(MediaType.APPLICATION_JSON); + return headers; + } + + /** + * Helper Method to create the Base64Value for headers + * + * @param userName + * @param password + * @return + */ + private String getBase64Value(String userName, String password) { + String authString = String.format("%s:%s", userName, password); + byte[] encodedAuthString = Base64.encodeBase64(authString.getBytes(Charset.forName(US_ASCII))); + return String.format(BASIC_AUTH, new String(encodedAuthString)); + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/IESService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IESService.java new file mode 100644 index 000000000..8ec7380a4 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IESService.java @@ -0,0 +1,111 @@ +package com.ingestpipeline.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ingestpipeline.model.TargetData; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.HttpHost; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + + +public interface IESService { + + /** + * Holds client for each indices. + */ + static Map esClient = new HashMap(); + static final String INVALID_QUERY_ERROR ="invalid query: must be boolean, match_phrase "; + + /** searches documents from ES based on query using RestHighLevelClient + * @param index - ElasticSearch Index + * @param searchQuery - which contains details for search + * @return + */ + JsonNode search(String index, ObjectNode searchQuery) throws IOException; + + Boolean push(TargetData requestBody) throws Exception; + + /** + * searches documents from ES based on query using restTemplate + * @param index + * @param query + * @return + * @throws Exception + */ + Map search(String index, String query) throws Exception; + + Boolean push(Map requestBody) throws Exception; + + + /** + * Translates a string query to SearchRequest + * valid query string has bool type query + * @param queryNode + * @return + */ + default SearchRequest buildSearchRequest(String index, ObjectNode queryNode) throws IOException{ + final BoolQueryBuilder query = QueryBuilders.boolQuery(); + + ArrayNode mustNodeArr = (ArrayNode) queryNode.get("query").get("bool").get("must"); + if(null == mustNodeArr) + throw new IllegalArgumentException(INVALID_QUERY_ERROR); + + mustNodeArr.elements().forEachRemaining(mustNode -> { + mustNode.fields().forEachRemaining(entry -> { + if(entry.getKey().equalsIgnoreCase("match_phrase") && entry.getValue().isObject()){ + JsonNode node = entry.getValue(); + Map.Entry dataNode = node.fields().next(); + query.must(QueryBuilders.matchPhraseQuery(dataNode.getKey(), dataNode.getValue().textValue())); + } + }); + + }); + return new SearchRequest(index).source(new SearchSourceBuilder().query(query)); + } + + /** + * + * @param indexName + * @param hostName + * @param port + * @param schema + * @return + */ + default RestHighLevelClient getClient(String indexName, String hostName, Integer port, String schema){ + RestHighLevelClient client = esClient.get(indexName); + if(null == client){ + client = new RestHighLevelClient(RestClient.builder(new HttpHost(hostName, port, schema))); + esClient.put(indexName, client); + } + return client; + } + + /*default boolean isIndexExists(String indexName) { + Response response; + try { + response = getClient(indexName).getLowLevelClient().performRequest(new Request("HEAD", "/" + indexName)); + return (200 == response.getStatusLine().getStatusCode()); + } catch (IOException e) { + return false; + } + + }*/ + + default boolean createIndex(String indexName){ + //TODO + return Boolean.TRUE; + } + + Map searchIndex(String index, String query) throws Exception; +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestService.java new file mode 100644 index 000000000..7b787bf4b --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestService.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.service; + +import java.io.File; + +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; + +@Service +public interface IngestService { + + Boolean ingestToPipeline(Object incomingData); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestServiceImpl.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestServiceImpl.java new file mode 100644 index 000000000..05ce88233 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/IngestServiceImpl.java @@ -0,0 +1,57 @@ +package com.ingestpipeline.service; + +import java.io.File; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; +import com.ingestpipeline.producer.IngestProducer; +import com.ingestpipeline.util.ApplicationProperties; +import com.ingestpipeline.util.Constants; + +/** + * This is a Service Implementation for all the actions which are with respect to Elastic Search + * @author Darshan Nagesh + * + */ +@Service(Constants.Qualifiers.INGEST_SERVICE) +public class IngestServiceImpl implements IngestService { + + public static final Logger LOGGER = LoggerFactory.getLogger(IngestServiceImpl.class); + + @Autowired + private IngestProducer ingestProducer; + + @Autowired + private ApplicationProperties applicationProperties; + + @Override + public Boolean ingestToPipeline(Object incomingData) { + LOGGER.info("Fetching the Incoming Data Config for the data received"); + String topic = ""; + String key = ""; + try { + if (applicationProperties.getPipelineRules().get(Constants.PipelineRules.VALIDATE_DATA)) { + topic = applicationProperties.getTransactionIngestTopic(); + key = applicationProperties.getTransactionIngestKey(); + } else if (applicationProperties.getPipelineRules().get(Constants.PipelineRules.TRANSFORM_DATA)) { + topic = applicationProperties.getTransactionValidationTopic(); + key = applicationProperties.getTransactionValidationKey(); + } else if (applicationProperties.getPipelineRules().get(Constants.PipelineRules.ENRICH_DATA)) { + topic = applicationProperties.getTransactionTransformationTopic(); + key = applicationProperties.getTransactionTransformationKey(); + } + } catch (Exception e) { + LOGGER.error("Encountered an Exception while Pushing the Data to pipeline on Ingest Service " + e.getMessage()); + ingestProducer.pushToPipeline(incomingData, Constants.KafkaTopics.ERROR_INTENT, Constants.KafkaTopics.ERROR_INTENT); + } + //System.out.println("incomingData 1559" + incomingData + "topic " + topic + "key " + key); + ingestProducer.pushToPipeline(incomingData, topic, key); + return true; + } + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformService.java new file mode 100644 index 000000000..84781d806 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformService.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.service; + +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; + +import java.util.Map; + +@Service +public interface TransformService { + + Boolean transformData(Map incomingData); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformServiceImpl.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformServiceImpl.java new file mode 100644 index 000000000..597c5eb8d --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/TransformServiceImpl.java @@ -0,0 +1,46 @@ +package com.ingestpipeline.service; + +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.bazaarvoice.jolt.Chainr; +import com.bazaarvoice.jolt.JsonUtils; +import com.ingestpipeline.util.Constants; + +/** + * This is a Service Implementation for all the actions which are with respect to Elastic Search + * @author Darshan Nagesh + * + */ +@Service(Constants.Qualifiers.TRANSFORM_SERVICE) +public class TransformServiceImpl implements TransformService { + + public static final Logger LOGGER = LoggerFactory.getLogger(TransformServiceImpl.class); + private static final String SEPARATOR = "_"; + private static final String JSON_EXTENSION = ".json"; + private static final String OBJECTIVE = "transform"; + private static final String CONFIGROOT = "config/"; + + @Override + public Boolean transformData(Map incomingData) { + String dataContext = incomingData.get(Constants.DATA_CONTEXT).toString(); + String dataContextVersion = incomingData.get(Constants.DATA_CONTEXT_VERSION).toString(); + String sourceUrl = CONFIGROOT.concat(OBJECTIVE.concat(SEPARATOR).concat(dataContext).concat(SEPARATOR).concat(dataContextVersion).concat(JSON_EXTENSION)); + List chainrSpecJSON = JsonUtils.jsonToList(this.getClass().getClassLoader().getResourceAsStream(sourceUrl)); + Chainr chainr = Chainr.fromSpec( chainrSpecJSON ); + Object inputJSON = incomingData.get(Constants.DATA_OBJECT); + try { + Object transformedOutput = chainr.transform( inputJSON ); + incomingData.put(Constants.DATA_OBJECT , transformedOutput); + return Boolean.TRUE; + } catch (Exception e) { + LOGGER.error("Encountered an error while tranforming the JSON : " + e.getMessage()); + return Boolean.FALSE; + } + } +} + \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationService.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationService.java new file mode 100644 index 000000000..d06e5cde2 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationService.java @@ -0,0 +1,14 @@ +package com.ingestpipeline.service; + +import org.springframework.stereotype.Service; + +import com.ingestpipeline.model.IncomingData; + +import java.util.Map; + +@Service +public interface ValidationService { + + Boolean validateData(Map incomingData); + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationServiceImpl.java b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationServiceImpl.java new file mode 100644 index 000000000..e211230ea --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/service/ValidationServiceImpl.java @@ -0,0 +1,73 @@ +package com.ingestpipeline.service; + +import java.util.HashMap; +import java.util.Map; + +import org.everit.json.schema.Schema; +import org.everit.json.schema.ValidationException; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.google.gson.GsonBuilder; +import com.ingestpipeline.util.Constants; + +/** + * This is a Service Implementation for all the actions which are with respect to Elastic Search + * @author Darshan Nagesh + * + */ +@Service(Constants.Qualifiers.VALIDATOR_SERVICE) +public class ValidationServiceImpl implements ValidationService { + + public static final Logger LOGGER = LoggerFactory.getLogger(ValidationServiceImpl.class); + private static final String OBJECTIVE = "validator"; + private static final String SEPARATOR = "_"; + private static final String JSON_EXTENSION = ".json"; + private static final String CONFIGROOT = "config/"; + private static Map schemaCache = new HashMap<>(); + + @Override + public Boolean validateData(Map incomingData) { + + //JSONObject jsonSubject = new JSONObject(new GsonBuilder().create().toJson(incomingData.getDataObject())); + JSONObject jsonSubject = new JSONObject(new GsonBuilder().create().toJson(incomingData.get("dataObject"))); + + try { + //getSchema(OBJECTIVE.concat(SEPARATOR).concat(incomingData.getDataContext()).concat(SEPARATOR).concat(incomingData.getDataContextVersion()) + getSchema(OBJECTIVE.concat(SEPARATOR).concat(incomingData.get("dataContext").toString()).concat(SEPARATOR).concat(incomingData.get("dataContextVersion").toString()) + .concat(JSON_EXTENSION)).validate(jsonSubject); + } catch (ValidationException ve) { + LOGGER.info("Validation Exception : " + ve.getMessage()); + return Boolean.FALSE; + } catch (Exception e) { + LOGGER.info("Validation Exception : " + e); + return Boolean.FALSE; + } + return Boolean.TRUE; + } + + private Schema getSchema(String location) { + Schema schema = schemaCache.get(location); + if (schema == null) { + try { + schema = loadSchema(location); + schemaCache.put(location, schema); + } catch(Exception e) { + LOGGER.error("Encountered an Exception while loading Schema : " + e.getMessage()); + } + } + return schema; + } + + private Schema loadSchema(String location) { + JSONObject jsonSchema = new JSONObject( + new JSONTokener(ClassLoader.getSystemClassLoader().getResourceAsStream(CONFIGROOT + location))); + return SchemaLoader.load(jsonSchema); + } + +} + \ No newline at end of file diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ApplicationProperties.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ApplicationProperties.java new file mode 100644 index 000000000..bb0b96def --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ApplicationProperties.java @@ -0,0 +1,162 @@ +package com.ingestpipeline.util; + +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.annotation.Order; +import org.springframework.core.env.Environment; + +@Configuration +@PropertySource(value = { "classpath:config/config.properties" }, ignoreResourceNotFound = true) +@Order(0) +public class ApplicationProperties { + + @Value("${kafka.transaction.ingest.topic}") + private String transactionIngestTopic; + + @Value("${kafka.transaction.ingest.key}") + private String transactionIngestKey; + + @Value("${kafka.transaction.validation.topic}") + private String transactionValidationTopic; + + @Value("${kafka.transaction.validation.key}") + private String transactionValidationKey; + + @Value("${kafka.transaction.transformation.topic}") + private String transactionTransformationTopic; + + @Value("${kafka.transaction.transformation.key}") + private String transactionTransformationKey; + + @Value("${kafka.transaction.enrichment.topic}") + private String transactionEnrichmentTopic; + + @Value("${kafka.transaction.enrichment.key}") + private String transactionEnrichmentKey; + + @Value("${schema.transaction.validation}") + private String transactionValidationSchema; + + @Value("${schema.transaction.transformation}") + private String transactionTransformSchema; + + @Value("${schema.transaction.enrichment}") + private String transactionEnrichmentSchema; + + @Value("${transformation.config.location}") + private static String transformationConfigLocations; + + @Value("#{${pipelinerules}}") + private Map pipelineRules; + + @Autowired + private Environment environment; + + public static String getTransformationConfigLocations() { + return transformationConfigLocations; + } + + public static void setTransformationConfigLocations(String transformationConfigLocations) { + ApplicationProperties.transformationConfigLocations = transformationConfigLocations; + } + + public String getTransactionValidationSchema() { + return transactionValidationSchema; + } + + public void setTransactionValidationSchema(String transactionValidationSchema) { + this.transactionValidationSchema = transactionValidationSchema; + } + + public String getTransactionTransformSchema() { + return transactionTransformSchema; + } + + public void setTransactionTransformSchema(String transactionTransformSchema) { + this.transactionTransformSchema = transactionTransformSchema; + } + + public String getTransactionEnrichmentSchema() { + return transactionEnrichmentSchema; + } + + public void setTransactionEnrichmentSchema(String transactionEnrichmentSchema) { + this.transactionEnrichmentSchema = transactionEnrichmentSchema; + } + + public Map getPipelineRules() { + return pipelineRules; + } + + public void setPipelineRules(Map pipelineRules) { + this.pipelineRules = pipelineRules; + } + + public String getTransactionIngestTopic() { + return transactionIngestTopic; + } + + public void setTransactionIngestTopic(String transactionIngestTopic) { + this.transactionIngestTopic = transactionIngestTopic; + } + + public String getTransactionIngestKey() { + return transactionIngestKey; + } + + public void setTransactionIngestKey(String transactionIngestKey) { + this.transactionIngestKey = transactionIngestKey; + } + + public String getTransactionValidationTopic() { + return transactionValidationTopic; + } + + public void setTransactionValidationTopic(String transactionValidationTopic) { + this.transactionValidationTopic = transactionValidationTopic; + } + + public String getTransactionValidationKey() { + return transactionValidationKey; + } + + public void setTransactionValidationKey(String transactionValidationKey) { + this.transactionValidationKey = transactionValidationKey; + } + + public String getTransactionTransformationTopic() { + return transactionTransformationTopic; + } + + public void setTransactionTransformationTopic(String transactionTransformationTopic) { + this.transactionTransformationTopic = transactionTransformationTopic; + } + + public String getTransactionTransformationKey() { + return transactionTransformationKey; + } + + public void setTransactionTransformationKey(String transactionTransformationKey) { + this.transactionTransformationKey = transactionTransformationKey; + } + + public String getTransactionEnrichmentTopic() { + return transactionEnrichmentTopic; + } + + public void setTransactionEnrichmentTopic(String transactionEnrichmentTopic) { + this.transactionEnrichmentTopic = transactionEnrichmentTopic; + } + + public String getTransactionEnrichmentKey() { + return transactionEnrichmentKey; + } + + public void setTransactionEnrichmentKey(String transactionEnrichmentKey) { + this.transactionEnrichmentKey = transactionEnrichmentKey; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ConfigLoader.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ConfigLoader.java new file mode 100644 index 000000000..0c482af53 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ConfigLoader.java @@ -0,0 +1,84 @@ +package com.ingestpipeline.util; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.core.io.support.ResourcePatternUtils; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Map; + +@Component("configLoader") +public class ConfigLoader { + + private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class); + private Map nameContentMap = new HashMap<>(); + @Autowired + private ResourceLoader resourceLoader; + public static final String RESOURCE_LOCATION = "classpath*:config/*.json"; + + + + /** + * Loads config resources + * @throws Exception + */ + public void loadResources() throws Exception { + Resource[] resources = getResources(RESOURCE_LOCATION); + + for (Resource resource : resources) { + String jsonContent = getContent(resource); + nameContentMap.put(resource.getFilename(), jsonContent); + } + logger.info("Number of resources loaded " + nameContentMap.size()); + + } + + /** + * To fetch a particular string content for a give resource/file name + * @param name + * @return + */ + public String get(String name) { + return nameContentMap.get(name); + } + + /** + * Loads all the resources/files with a given pattern *.json + * @param pattern path with *json + * @return + * @throws IOException + */ + private Resource[] getResources(String pattern) throws IOException { + Resource[] resources = ResourcePatternUtils.getResourcePatternResolver(resourceLoader).getResources(pattern); + return resources; + } + + /** + * Returns a content of resource + * + * @param resource + * @return + */ + private String getContent(Resource resource) { + String content = null; + try { + InputStream is = resource.getInputStream(); + byte[] encoded = IOUtils.toByteArray(is); + content = new String(encoded, Charset.forName("UTF-8")); + + } catch (IOException e) { + logger.error("Cannot load resource " + resource.getFilename()); + + } + return content; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/Constants.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/Constants.java new file mode 100644 index 000000000..f99a0cf87 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/Constants.java @@ -0,0 +1,66 @@ +package com.ingestpipeline.util; + +/** + * Constants which are with respect to the Ingest App + * + * @author Darshan Nagesh + * + */ + +public interface Constants { + public interface Paths { + final String ELASTIC_PUSH_CONTROLLER_PATH = "/ingest"; + final String SAVE = "/save"; + final String UPLOAD = "/upload"; + final String Targets = "/targets"; + final String Collections = "/getCollections"; + final String ES_INDEX = "/migrate/{indexName}"; + } + + public interface Qualifiers { + final String INGEST_SERVICE = "ingestService"; + final String VALIDATOR_SERVICE = "validatorService"; + final String TRANSFORM_SERVICE = "transformService"; + final String ENRICHMENT_SERVICE = "enrichmentService"; + } + + public static String SUCCESS = "success"; + public static int UNAUTHORIZED_ID = 401; + public static int SUCCESS_ID = 200; + public static int FAILURE_ID = 320; + public static String UNAUTHORIZED = "Invalid credentials. Please try again."; + public static String PROCESS_FAIL = "Process failed, Please try again."; + public static String DATE_FORMAT = "yyyy.MM.dd G 'at' HH:mm:ss z"; + public static String INDIAN_TIMEZONE = "IST"; + + public static String ALLOWED_METHODS_GET = "GET"; + public static String ALLOWED_METHODS_POST = "POST"; + + public interface KafkaTopics { + public static final String INGEST_DATA = "ingestData"; + public static final String VALID_DATA = "validData"; + public static final String TRANSFORMED_DATA = "transformedData"; + public static final String ERROR_INTENT = "DataError"; + } + + public interface BeanContainerFactory { + public static final String INCOMING_KAFKA_LISTENER = "incomingKafkaListenerContainerFactory"; + } + + public interface PipelineRules { + public static final String VALIDATE_DATA = "VALIDATE"; + public static final String TRANSFORM_DATA = "TRANSFORM"; + public static final String ENRICH_DATA = "ENRICH"; + } + + public static String DATA_CONTEXT = "dataContext"; + public static String DATA_CONTEXT_VERSION = "dataContextVersion"; + public static String DATA_OBJECT = "dataObject"; + + public static String ERROR_IN_PIPEINE = "errorPipeline"; + + public static int HEADER_ROW = 1; + public static String MUNICIPAL_CORPORATIONS = "Municipal Corporations"; + public static String ES_INDEX_COLLECTION = "collectionsindex-v1"; + public static String ES_INDEX_BILLING = "billingservice"; +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/CustomErrorType.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/CustomErrorType.java new file mode 100644 index 000000000..687121237 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/CustomErrorType.java @@ -0,0 +1,16 @@ +package com.ingestpipeline.util; + + +public class CustomErrorType { + + private String errorMessage; + + public CustomErrorType(String errorMessage){ + this.errorMessage = errorMessage; + } + + public String getErrorMessage() { + return errorMessage; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtil.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtil.java new file mode 100644 index 000000000..de0a31d8e --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtil.java @@ -0,0 +1,94 @@ +package com.ingestpipeline.util; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.gson.Gson; + +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; + +public class JSONUtil { + @Autowired + public ObjectMapper mapper; + @Autowired + public Gson gson; + + /** + * Field value to replace by new text. Replace node by given text to Parent's + * hierarchy. Field will not be added if not found existing already + * + * @param parent + * @param fieldName + * @param newValue + */ + public static void replaceFieldValue(ObjectNode parent, String fieldName, String newValue) { + if (parent.has(fieldName)) { + parent.put(fieldName, newValue); + } + parent.fields().forEachRemaining(entry -> { + JsonNode entryValue = entry.getValue(); + if (entryValue.isArray()) { + for (int i = 0; i < entryValue.size(); i++) { + if (entry.getValue().get(i).isObject()) + replaceFieldValue((ObjectNode) entry.getValue().get(i), fieldName, newValue); + } + } else if (entryValue.isObject()) { + replaceFieldValue((ObjectNode) entry.getValue(), fieldName, newValue); + } + }); + } + + /** + * Finds the value for a given key from the tree + * @param parent the tree + * @param key the field name + * @param value the value for the given field + */ + public static void findValue(ObjectNode parent, String key, StringBuffer value) { + parent.fields().forEachRemaining(entry -> { + if(entry.getKey().equalsIgnoreCase(key)){ + value.append(entry.getValue().asText()); + } else { + JsonNode entryValue = entry.getValue(); + if(entryValue.isArray()){ + for (int i=0; i < entryValue.size(); i++){ + if (entry.getValue().get(i).isObject()) + findValue((ObjectNode) entry.getValue().get(i), key, value); + } + } else if (entryValue.isObject()){ + findValue((ObjectNode) entry.getValue(), key, value); + } + } + }); + } + + + /** + * @return + */ + public static String getJsonString(ObjectMapper objectMapper,Object object) throws JsonProcessingException { + if(objectMapper != null){ + return objectMapper.writeValueAsString(object); + } + return null; + } + + public ObjectMapper getMapper() { + return mapper; + } + + public void setObjectMapper(ObjectMapper objectMapper){ + mapper=objectMapper; + } + + public Gson getGson() { + return gson; + } + + public void setGson(Gson gsonn) + { + gson = gsonn; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtils.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtils.java new file mode 100644 index 000000000..b317e8be0 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JSONUtils.java @@ -0,0 +1,16 @@ +package com.ingestpipeline.util; + +import org.json.JSONObject; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonPropertyOrder({ "dataContext", "dataContextVersion", "dataObject"}) +public class JSONUtils { + public JSONObject JSONWrapper(JSONObject json) { + JSONObject obj = new JSONObject(); + obj.put("dataContext", "target"); + obj.put("dataContextVersion", "v1"); + obj.put("dataObject", json); + return obj; + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/JsonKey.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JsonKey.java new file mode 100644 index 000000000..27dcd8a32 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/JsonKey.java @@ -0,0 +1,20 @@ +/** + * + */ +package com.ingestpipeline.util; + +/** + * @author Abhishek + * + */ +public class JsonKey { + + public static final String STATUS_CODE = "statusCode"; + public static final String STATUS = "statusInfo"; + public static final String STATUS_MESSAGE = "statusMessage"; + public static final String ERROR_MESSAGE = "errorMessage"; + + public static final String RESPONSE_DATA = "responseData"; + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/MockMultipartFile.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/MockMultipartFile.java new file mode 100644 index 000000000..f678698ef --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/MockMultipartFile.java @@ -0,0 +1,77 @@ +package com.ingestpipeline.util; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +import org.springframework.web.multipart.MultipartFile; + +public class MockMultipartFile implements MultipartFile { + + private String name; + private String originalFileName; + private String contentType; + private byte[] content; + + public MockMultipartFile(String name, String originalFileName, String contentType, byte[] content) { + super(); + this.name = name; + this.originalFileName = originalFileName; + this.contentType = contentType; + this.content = content; + } + + @Override + public String getName() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getOriginalFilename() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getContentType() { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean isEmpty() { + // TODO Auto-generated method stub + return false; + } + + @Override + public long getSize() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public byte[] getBytes() throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InputStream getInputStream() throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void transferTo(File dest) throws IOException, IllegalStateException { + // TODO Auto-generated method stub + + } + + + + + + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ReadUtil.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ReadUtil.java new file mode 100644 index 000000000..1fa86e44f --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ReadUtil.java @@ -0,0 +1,347 @@ +package com.ingestpipeline.util; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.DateFormat; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Scanner; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.fileupload.disk.DiskFileItem; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.DateUtil; +import org.apache.poi.ss.usermodel.RichTextString; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.ss.util.CellRangeAddress; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.commons.CommonsMultipartFile; + +import com.ingestpipeline.controller.RestApiController; +import com.ingestpipeline.service.IngestService; + +@Component("readUtil") +public class ReadUtil { + + public static final Logger LOGGER = LoggerFactory.getLogger(ReadUtil.class); + private static String UPLOADED_FOLDER = ""; + public static Path path; + public static File uploadFile = new File( + System.getProperty("user.dir") + System.getProperty("file.separator") + "uploads"); + + public static JSONArray getFiletoDirectory(MultipartFile file) throws Exception { + byte[] bytes = file.getBytes(); + if (!uploadFile.exists()) { + uploadFile.mkdir(); + } + UPLOADED_FOLDER = uploadFile.toString(); + path = Paths.get(UPLOADED_FOLDER + System.getProperty("file.separator") + file.getOriginalFilename()); + Files.write(path, bytes); + JSONArray fileIntoJsonArray = readFilefromDirectory(); + String filename = null; + Path p = Paths.get(file.getOriginalFilename()); + if (p.toString().indexOf(".") > 0) { + filename = p.toString().substring(0, p.toString().lastIndexOf(".")); + } + String jsonArrayFileName = filename + ".json"; + writeJsonArrayToFile(fileIntoJsonArray, jsonArrayFileName); + return fileIntoJsonArray; + } + + private static JSONArray readFilefromDirectory() throws Exception { + String workbookSheetName = null; + int workbookSheetIndex = -1; + String getFileExtension = FilenameUtils.getExtension(path.toString()); + Workbook workbook = null; + if (getFileExtension.endsWith("xlsx")) { + workbook = new XSSFWorkbook(); + workbook = WorkbookFactory.create(new File(path.toString())); + } else { + throw new Exception("invalid file, should be xlsx"); + } + + JSONArray workbookToJsonArray = new JSONArray(); + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + Sheet sheet = workbook.getSheetAt(i); + workbookSheetName = sheet.getSheetName(); + workbookSheetIndex = workbook.getSheetIndex(workbookSheetName); + if (workbookSheetName != null && workbookSheetName.length() > 0) { + workbookToJsonArray = workbookToJsonArray.put(getSheetToJsonObject(workbook, sheet)); + } + } + + JSONArray bookInJsonArray = new JSONArray(); + for (int i = 0; i < workbookToJsonArray.length(); i++) { + JSONArray sheetInJsonArray = new JSONArray(); + sheetInJsonArray = (JSONArray) workbookToJsonArray.get(i); + Iterator itr = sheetInJsonArray.iterator(); + while (itr.hasNext()) { + JSONObject obj = (JSONObject) itr.next(); + bookInJsonArray.put(obj); + } + } + return bookInJsonArray; + } + + private static JSONArray getSheetToJsonObject(Workbook workbook, Sheet sheet) { + String workbookSheetName = sheet.getSheetName(); + int workbookSheetIndex = workbook.getSheetIndex(workbookSheetName); + Object financialYear = null; + int firstRowNum = sheet.getFirstRowNum(), lastRowNum = sheet.getLastRowNum(), + ulbFirstRowNumber = Constants.HEADER_ROW + 2, ulbDestRowNumber = -1; + Row row = null; + int rowFirstCellNum = -1, rowLastCellNum = -1; + List> rowRecordList = new LinkedList>(); + Map> rowRecordMap = new LinkedHashMap>(); + Map> municipalCity = new LinkedHashMap>(); + + if (lastRowNum > 0) { + for (int i = firstRowNum; i <= lastRowNum; i++) { + row = sheet.getRow(i); + if (row != null) { + rowFirstCellNum = row.getFirstCellNum(); + rowLastCellNum = sheet.getRow(i).getLastCellNum(); + List rowRecord = new LinkedList(); + Map> singleRowDataMap = new LinkedHashMap>(); + for (int j = rowFirstCellNum; j < rowLastCellNum; j++) { + Cell cell = row.getCell(j); + if (isMergedRegion(workbook, sheet, i, j)) { + sheet = getSheetMergerCellRegion(workbook, sheet, i, j); + } + singleRowDataMap.put(i, getRowRecord(cell, rowRecord)); + } + + if (!rowRecord.isEmpty()) { + rowRecordList.add(rowRecord); + // rowRecordMap.put(i, rowRecord); + rowRecordMap.putAll(singleRowDataMap); + + } + + if (rowRecordList.size() == 1) { + financialYear = rowRecordList.get(Constants.HEADER_ROW - 1).get(3).toString().trim(); + } + } + } + } + + List sheetHeaderList = new LinkedList(); + List customHeaderList = new LinkedList(); + Map customHeaderMap = new LinkedHashMap(); + JSONArray getMunicipalCityToJsonArray = new JSONArray(); + Map> lastRowRecord = new LinkedHashMap>(); + customHeaderList.add("Sheet Name"); + customHeaderList.add("Financial Year"); + customHeaderList.add("Timestamp"); + // customHeaderList.add("Municipal Corporations"); + customHeaderMap.put(customHeaderList.get(0), workbookSheetName); + customHeaderMap.put(customHeaderList.get(1), financialYear); + DateFormat df = new SimpleDateFormat("dd/MM/yy HH:mm:S"); + Date dateobj = new Date(); + customHeaderMap.put(customHeaderList.get(2), df.format(dateobj)); + JSONObject customHeaderJsonObject = new JSONObject(); + JSONArray municipalCitiesIntoJsonArray = new JSONArray(); + for (Map.Entry> itrRowRecordMap : rowRecordMap.entrySet()) { + if (itrRowRecordMap.getKey() == Constants.HEADER_ROW) { + sheetHeaderList.addAll(itrRowRecordMap.getValue()); + } + if (itrRowRecordMap.getKey() >= ulbFirstRowNumber) { + JSONObject municipalCitiesIntoJsonObject = new JSONObject(); + Map mc = new LinkedHashMap(); + municipalCity.put(itrRowRecordMap.getKey(), itrRowRecordMap.getValue()); + for (Map.Entry itrCustomHeaderMap : customHeaderMap.entrySet()) { + municipalCitiesIntoJsonObject.accumulate(itrCustomHeaderMap.getKey().toString(), + itrCustomHeaderMap.getValue()); + } + + for (int i = 0; i < sheetHeaderList.size(); i++) { + if ((!sheetHeaderList.get(i).toString().contentEquals("-")) + && (!itrRowRecordMap.getValue().get(i).toString().contentEquals("-"))) { + + municipalCitiesIntoJsonObject.put(sheetHeaderList.get(i).toString(), + itrRowRecordMap.getValue().get(i)); + } + } + + municipalCitiesIntoJsonArray.put(municipalCitiesIntoJsonObject); + } + } + return municipalCitiesIntoJsonArray; + } + + private static List getRowRecord(Cell cell, List rowRecord) { + switch (cell.getCellType()) { + case STRING: + String str = cell.getRichStringCellValue().getString(); + str = str.replaceAll("(\\n\\s)+|(\\r\\n\\s)+|(\\r\\s)+|(\\r)+|(\\n)+|(\\s)+", " "); + rowRecord.add(str); + break; + case NUMERIC: + if (DateUtil.isCellDateFormatted(cell)) { + rowRecord.add(cell.getDateCellValue()); + break; + } else { + Double d = cell.getNumericCellValue(); + DecimalFormat numberFormat = new DecimalFormat("#.00"); + rowRecord.add(Double.parseDouble(numberFormat.format(d))); + } + break; + case BOOLEAN: + rowRecord.add(cell.getBooleanCellValue()); + break; + case FORMULA: + if (cell.getCachedFormulaResultType().equals(CellType.NUMERIC)) { + Double d = cell.getNumericCellValue() * 100; + DecimalFormat numberFormat = new DecimalFormat("#.00"); + rowRecord.add(Double.parseDouble(numberFormat.format(d))); + break; + } else if (cell.getCachedFormulaResultType().equals(CellType.STRING)) { + rowRecord.add(cell.getNumericCellValue()); + break; + } + break; + case BLANK: + rowRecord.add("-"); + break; + default: + rowRecord.add("no type match"); + break; + } + return rowRecord; + } + + private static boolean isMergedRegion(Workbook workbook, Sheet sheet, int cellRow, int cellColumn) { + int retVal = 0; + int sheetMergerCount = sheet.getNumMergedRegions(); + for (int i = 0; i < sheetMergerCount; i++) { + CellRangeAddress region = sheet.getMergedRegion(i); + int firstRow = region.getFirstRow(), firstCol = region.getFirstColumn(), lastRow = region.getLastRow(), + lastCol = region.getLastColumn(); + if (cellRow >= firstRow && cellRow <= lastRow) { + if (cellColumn >= firstCol && cellColumn <= lastCol) { + retVal = lastCol - firstCol + 1; + if (retVal > 0) { + return true; + } + break; + } + } + } + return false; + } + + private static Sheet getSheetMergerCellRegion(Workbook workbook, Sheet sheet, int cellRow, int cellColumn) { + int retVal = 0; + int sheetMergerCount = sheet.getNumMergedRegions(); + for (int i = 0; i < sheetMergerCount; i++) { + CellRangeAddress region = sheet.getMergedRegion(i); + int firstRow = region.getFirstRow(); + int firstCol = region.getFirstColumn(); + int lastRow = region.getLastRow(); + int lastCol = region.getLastColumn(); + if (cellRow >= firstRow && cellRow <= lastRow) { + if (cellColumn >= firstCol && cellColumn <= lastCol) { + retVal = lastCol - firstCol + 1; + Row row = sheet.getRow(i); + if (retVal > 0) { + for (int j = firstRow; j <= lastRow; j++) { + for (int k = firstCol; k <= lastCol; k++) { + Cell cell = sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()); + String stringValue1 = null; + Double doubleValue1 = -1.00; + Date dateValue1 = new Date(); + Boolean booleanValue1 = false; + RichTextString formulaValue1 = null; + switch (cell.getCellType()) { + case STRING: + stringValue1 = sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()) + .getRichStringCellValue().getString(); + sheet.getRow(j).getCell(k).setCellValue(stringValue1); + break; + case NUMERIC: + if (DateUtil.isCellDateFormatted( + sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()))) { + dateValue1 = sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()) + .getDateCellValue(); + sheet.getRow(j).getCell(k).setCellValue(dateValue1); + } else { + doubleValue1 = sheet.getRow(region.getFirstRow()) + .getCell(region.getFirstColumn()).getNumericCellValue(); + sheet.getRow(j).getCell(k).setCellValue(doubleValue1); + } + break; + case BOOLEAN: + booleanValue1 = sheet.getRow(region.getFirstRow()).getCell(region.getFirstColumn()) + .getBooleanCellValue(); + sheet.getRow(j).getCell(k).setCellValue(booleanValue1); + break; + case FORMULA: + if (cell.getCachedFormulaResultType().equals(CellType.NUMERIC)) { + doubleValue1 = cell.getNumericCellValue() * 100; + sheet.getRow(j).getCell(k).setCellValue(doubleValue1); + break; + } else if (cell.getCachedFormulaResultType().equals(CellType.STRING)) { + formulaValue1 = cell.getRichStringCellValue(); + sheet.getRow(j).getCell(k).setCellValue(formulaValue1); + break; + } + break; + case BLANK: + break; + default: + } + } + } + } + } + } + } + return sheet; + } + + private static void writeJsonArrayToFile(JSONArray data, String fileName) throws IOException { + String currentWorkingFolder = System.getProperty("user.dir"), + filePathSeperator = System.getProperty("file.separator"), + filePath = currentWorkingFolder + filePathSeperator + fileName; + BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(new File(filePath))); + bufferedWriter.write(data.toString()); + bufferedWriter.flush(); + bufferedWriter.close(); + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseCode.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseCode.java new file mode 100644 index 000000000..fe20fbf7f --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseCode.java @@ -0,0 +1,85 @@ +package com.ingestpipeline.util; + +/** + * + * @author Abhishek + * + */ +public enum ResponseCode { + UnAuthorised(Constants.UNAUTHORIZED_ID, Constants.UNAUTHORIZED), Success( + Constants.SUCCESS_ID, Constants.SUCCESS),FAILURE( + Constants.FAILURE_ID, Constants.PROCESS_FAIL); + /** + * error code contains int value + */ + private int errorCode; + /** + * errorMessage contains proper error message. + */ + private String errorMessage; + + + + /** + * @param errorCode + * @param errorMessage + */ + private ResponseCode(int errorCode, String errorMessage) { + this.errorCode = errorCode; + this.errorMessage = errorMessage; + } + + /** + * + * @param errorCode + * @return + */ + public String getMessage(int errorCode) { + return ""; + } + + /** + * @return + */ + public int getErrorCode() { + return errorCode; + } + + /** + * @param errorCode + */ + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + /** + * @return + */ + public String getErrorMessage() { + return errorMessage; + } + + /** + * @param errorMessage + */ + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + + /** + * This method will provide status message based on code + * + * @param code + * @return String + */ + public static String getResponseMessage(int code) { + String value = ""; + ResponseCode responseCodes[] = ResponseCode.values(); + for (ResponseCode actionState : responseCodes) { + if (actionState.getErrorCode() == code) { + value = actionState.getErrorMessage(); + } + } + return value; + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseGenerator.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseGenerator.java new file mode 100644 index 000000000..46b73508c --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseGenerator.java @@ -0,0 +1,79 @@ +package com.ingestpipeline.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class ResponseGenerator { + + + private static ObjectMapper objectMapper = new ObjectMapper(); + + public static String failureResponse() throws JsonProcessingException{ + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + return JSONUtil.getJsonString(objectMapper,response); + } + + + public static String failureResponse(String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE,message); + actualResponse.putPOJO(JsonKey.STATUS,response); + + return JSONUtil.getJsonString(objectMapper,actualResponse); + } + + public static String feedbackFailureResponse(String message) throws JsonProcessingException{ + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.FAILURE.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, + ResponseCode.FAILURE.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE,message); + actualResponse.putPOJO(JsonKey.STATUS,response); + + return JSONUtil.getJsonString(objectMapper,actualResponse); + } + + public static String successResponse(Object obj) throws JsonProcessingException { + ObjectNode actualResponse = objectMapper.createObjectNode(); + + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, ResponseCode.Success.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, ""); + actualResponse.putPOJO(JsonKey.STATUS,response); + if (obj != null) { + actualResponse.putPOJO(JsonKey.RESPONSE_DATA, obj); + } + + return JSONUtil.getJsonString(objectMapper,actualResponse); + } + + public static String feedbackSuccessResponse(Object obj) throws JsonProcessingException { + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, ResponseCode.Success.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, ""); + return JSONUtil.getJsonString(objectMapper,response); + } + + public static String feedbackSuccessResponse(String message) throws JsonProcessingException { + ObjectNode response = objectMapper.createObjectNode(); + response.put(JsonKey.STATUS_CODE, ResponseCode.Success.getErrorCode()); + response.put(JsonKey.STATUS_MESSAGE, ResponseCode.Success.getErrorMessage()); + response.put(JsonKey.ERROR_MESSAGE, ""); + return JSONUtil.getJsonString(objectMapper, response); + } +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseMessages.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseMessages.java new file mode 100644 index 000000000..5f75d4415 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseMessages.java @@ -0,0 +1,11 @@ +package com.ingestpipeline.util; + +public interface ResponseMessages { + + public interface ErrorMessages { + } + + public interface SuccessMessages { + } + +} diff --git a/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseUtil.java b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseUtil.java new file mode 100644 index 000000000..697a1de95 --- /dev/null +++ b/dashboard-ingest/src/main/java/com/ingestpipeline/util/ResponseUtil.java @@ -0,0 +1,91 @@ +package com.ingestpipeline.util; + +import java.io.IOException; + +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * + * @author Abhishek + * + */ + +public class ResponseUtil { + + /** + * Method to throw bad request with error message + * + * @param errorDescription + */ + public static void sendBadRequest(String errorDescription) { + ResponseBuilder resp = Response.status(Response.Status.BAD_REQUEST); + resp.entity(errorDescription); + WebApplicationException ex = new WebApplicationException(resp.build()); + throw ex; + } + + /** + * Method to throw Unauthorized request with error message + * + * @param errorDescription + *//* + public static Response sendUnauthorized(String errorDescription) throws JsonProcessingException{ + ResponseBuilder resp = Response.status(Response.Status.UNAUTHORIZED); + resp.entity(ResponseGenerator.failureResponse(errorDescription)); + return resp.build(); + }*/ + + /** + * Method to throw Internal server error + * + * @param errorDescription + */ + public static Response sendServerError(String errorDescription) { + ResponseBuilder resp = Response.status(Response.Status.INTERNAL_SERVER_ERROR); + resp.entity(errorDescription); + return resp.build(); + } + + /** + * Method to throw Unauthorized request with error message + * + * @param errorDescription + */ + public static void unauthorizedResponse(HttpServletResponse response, String errorDescription) throws IOException { + response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.setContentType("application/json"); + response.getWriter().write(errorDescription); + } + + public static Response sendOK(ObjectNode obj) { + ResponseBuilder resp = Response.status(Response.Status.OK); + resp.entity(obj); + return resp.build(); + } + + public static Response sendOK(String obj) { + ResponseBuilder resp = Response.status(Response.Status.OK); + resp.entity(obj); + return resp.build(); + } + + /*public static void sendInternalError(String errorDescription) throws JsonProcessingException { + ResponseBuilder resp = Response.status(Response.Status.INTERNAL_SERVER_ERROR); + resp.entity(ResponseGenerator.failureResponse(errorDescription)); + WebApplicationException ex = new WebApplicationException(resp.build()); + throw ex; + } + + public static void sendInternalError() throws JsonProcessingException { + ResponseBuilder resp = Response.status(Response.Status.INTERNAL_SERVER_ERROR); + resp.entity(ResponseGenerator.failureResponse()); + WebApplicationException ex = new WebApplicationException(resp.build()); + throw ex; + }*/ +} diff --git a/dashboard-ingest/src/main/resources/application.properties b/dashboard-ingest/src/main/resources/application.properties new file mode 100644 index 000000000..aca35a417 --- /dev/null +++ b/dashboard-ingest/src/main/resources/application.properties @@ -0,0 +1,69 @@ +#--------------------------- PATH & PORT CONFIGURATIONS ---------------------------# + +# SET CONTEXT PATH +server.port=8082 + +# KAFKA SERVER CONFIGURATIONS +kafka.config.bootstrap_server_config=localhost:9092 + +spring.kafka.consumer.value-deserializer=com.ingestpipeline.consumer.HashMapDeserializer +spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer +spring.kafka.consumer.group-id=transaction-chain +spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer +spring.kafka.producer.value-serializer=org.springframework.kafka.support.serializer.JsonSerializer + +# KAFKA CONSUMER CONFIGURATIONS +kafka.consumer.config.auto_commit=true +kafka.consumer.config.auto_commit_interval=100 +kafka.consumer.config.session_timeout=15000 +kafka.consumer.config.group_id=pipeline-group +kafka.consumer.config.auto_offset_reset=earliest + +# KAFKA PRODUCER CONFIGURATIONS +kafka.producer.config.retries_config=0 +kafka.producer.config.batch_size_config=100000 +kafka.producer.config.linger_ms_config=100 +kafka.producer.config.buffer_memory_config=66554432 + +#----------------------------- ELASTIC SEARCH CONFIGURATIONS ------------------------------# +services.esindexer.host=http://104.211.240.72:9200/ +services.esindexer.host.name=104.211.240.72 +services.esindexer.host.port=9200 +services.esindexer.username=deploy +services.esindexer.password=Deploy123 +es.target.index.name = dss-target_v1 +#es.index.name=col-dss-v1 +es.index.name=dss-collection_v1 +es.document.type=pipeline_data +es.host.schema=https +es.index.searchQuery.collection = {\"size\": \"200\",\"query\":{\"bool\":{\"must\":[{\"wildcard\": {\"Data.tenantId.keyword\":\"pb.*\"}}]}}} +es.index.searchQuery.billing = {\"size\": \"200\",\"query\":{\"bool\":{\"must\":[{\"wildcard\": {\"tenantid.keyword\":\"pb.*\"}}]}}} +egov.services.esindexer.host.name=http://104.211.240.72:9200/ +egov.services.esindexer.host.search=/_search + +spring.data.elasticsearch.cluster-name=elasticsearch-v1 +spring.data.elasticsearch.cluster-nodes=elasticsearch-v1 + +#----------------------------- Pipeline Schema Config Locations ------------------------------# +transformation.config.location=config/transform_transaction_v1.json + +schema.transaction.validation=config/validator_transaction_v1.json +schema.transaction.transformation=config/transform_transaction_v1.json +schema.transaction.enrichment=config/enrichment_transaction_v1.json + +#------------------------------ File Import CONFIGURATIONS ------------------------------------# +spring.servlet.multipart.enabled=true + +#------------------------------ H2 ------------------------------------------------------------# +spring.h2.console.enabled=true +spring.h2.console.path=/h2-console + +spring.datasource.url=jdbc:h2:mem:testdb +spring.datasource.driverClassName=org.h2.Driver +spring.datasource.username=sa +spring.datasource.password=password +spring.jpa.database-platform=org.hibernate.dialect.H2Dialect +spring.jpa.show-sql=true +#spring.datasource.url=jdbc:h2:file:/d1/demo + + diff --git a/dashboard-ingest/src/main/resources/config/CollectionDomainConfig.json b/dashboard-ingest/src/main/resources/config/CollectionDomainConfig.json new file mode 100644 index 000000000..47fd481e8 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/CollectionDomainConfig.json @@ -0,0 +1,51 @@ +{ + "domainConfig": [ + { + "id": "1", + "businessType": "PT", + "indexName": "ptindex-v1", + "documentType": "", + "query": "{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"Data.propertyDetails.assessmentNumber\":\"value\"}},{\"match_phrase\":{\"Data.propertyId\":\"value\"}}]}}}", + "collectionRef": [ + { + "fieldName": "consumerCode", + "argument": "Data.propertyId", + "dataType": "String" + }, + { + "fieldName": "consumerCode", + "argument": "Data.propertyDetails.assessmentNumber", + "dataType": "String" + } + ] + }, + { + "id": "2", + "businessType": "TL", + "indexName": "tlindex-v1", + "documentType": "", + "query": "{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"Data.tradelicense.applicationnumber\":\"value\"}}]}}}", + "collectionRef": [ + { + "fieldName": "consumerCode", + "argument": "Data.tradelicense.applicationnumber", + "dataType": "String" + } + ] + }, + { + "id": "3", + "businessType": "PGR", + "indexName": "pgrindex-v1", + "documentType": "", + "query": "{}", + "collectionRef": [ + { + "fieldName": "consumerCode", + "argument": "complainNumber", + "dataType": "String" + } + ] + } + ] +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/resources/config/config.properties b/dashboard-ingest/src/main/resources/config/config.properties new file mode 100644 index 000000000..fe268d9f1 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/config.properties @@ -0,0 +1,18 @@ +#--------------------------- Kafka Topics & Configurations ---------------------------# +dataContext=transaction + +kafka.transaction.ingest.topic=ingestData +kafka.transaction.ingest.key=ingestKey + +kafka.transaction.validation.topic=validData +kafka.transaction.validation.key=validKey + +kafka.transaction.transformation.topic=transformedData +kafka.transaction.transformation.key=transformedKey + +kafka.transaction.enrichment.topic=enrichedData +kafka.transaction.enrichment.key=enrichedKey + +pipelinerules={'VALIDATE':false,'TRANSFORM':true,'ENRICH':true} + +transformation.config.location=config/transform_transaction_v1.json diff --git a/dashboard-ingest/src/main/resources/config/enrichment_transaction_v1.json b/dashboard-ingest/src/main/resources/config/enrichment_transaction_v1.json new file mode 100644 index 000000000..038c1be33 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/enrichment_transaction_v1.json @@ -0,0 +1,49 @@ +{ + "dataCollectorName": "UserInformation", + "collectionSource": "UserService", + "basePath": "http://rain.idc.tarento.com/user/", + "collectionPoints": [ + { + "name": "UserMetaInfo", + "extensionPath": "/getUserMetaInfo", + "paramsPassed": [ + { + "sourceParam": "Transaction.KioskOperator.Id", + "pathLabel": "id", + "dataType": "Long", + "defaultValue": "0" + }, + { + "sourceParam": null, + "pathLabel": "isActive", + "dataType": "Boolean", + "defaultValue": "TRUE" + } + ], + "responseStructure": "{}", + "responseMapping": [ + { + + } + ] + }, + { + "name": "UserRoleInfo", + "extensionPath": "/getUserRole", + "paramsPassed": [ + { + "sourceParam": "Transaction.KioskOperator.Id", + "pathLabel": "userId", + "dataType": "Long", + "defaultValue": "0" + }, + { + "sourceParam": null, + "pathLabel": "isActive", + "dataType": "Boolean", + "defaultValue": "TRUE" + } + ] + } + ] +} diff --git a/dashboard-ingest/src/main/resources/config/transform_collection_v1.json b/dashboard-ingest/src/main/resources/config/transform_collection_v1.json new file mode 100644 index 000000000..d0a324d5a --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_collection_v1.json @@ -0,0 +1,102 @@ +[ + { + "operation": "shift", + "spec": { + "tenantId": "tenantId", + "transactionId": "transactionId", + "Bill": { + "*": { + "billDetails": { + "*": { + "voucherHeader": "Bill.billDetails.voucherHeader", + "boundary": "Bill.billDetails.boundary", + "collectedAmount": "Bill.billDetails.collectedAmount", + "channel": "Bill.billDetails.channel", + "manualReceiptDate": "Bill.billDetails.manualReceiptDate", + "expiryDate": "Bill.billDetails.expiryDate", + "demandId": "Bill.billDetails.demandId", + "amountPaid": "Bill.billDetails.amountPaid", + "consumerCode": "Bill.billDetails.consumerCode", + "id": "Bill.billDetails.id", + "department": "Bill.billDetails.department", + "receiptNumber": "Bill.billDetails.receiptNumber", + "businessService": "Bill.billDetails.businessService", + "receiptType": "Bill.billDetails.receiptType", + "fromPeriod": "Bill.billDetails.fromPeriod", + "stateId": "Bill.billDetails.stateId", + "receiptDate": "Bill.billDetails.receiptDate", + "bill": "Bill.billDetails.bill", + "billDate": "Bill.billDetails.billDate", + "consumerType": "Bill.billDetails.consumerType", + "totalAmount": "Bill.billDetails.totalAmount", + "collectionType": "Bill.billDetails.collectionType", + "fund": "Bill.billDetails.fund", + "manualReceiptNumber": "Bill.billDetails.manualReceiptNumber", + "tenantId": "Bill.billDetails.tenantId", + "toPeriod": "Bill.billDetails.toPeriod", + "partPaymentAllowed": "Bill.billDetails.partPaymentAllowed", + "billNumber": "Bill.billDetails.billNumber", + "status": "Bill.billDetails.status", + "billAccountDetails": { + "*": { + "taxHeadCode": "Bill.billDetails.billAccountDetails.[&1].taxHeadCode", + "glcode": "Bill.billDetails.billAccountDetails.[&1].glcode", + "amount": "Bill.billDetails.billAccountDetails.[&1].amount", + "billDetail": "Bill.billDetails.billAccountDetails.[&1].billDetail", + "purpose": "Bill.billDetails.billAccountDetails.[&1].purpose", + "adjustedAmount": "Bill.billDetails.billAccountDetails.[&1].adjustedAmount", + "tenantId": "Bill.billDetails.billAccountDetails.[&1].tenantId", + "id": "Bill.billDetails.billAccountDetails.[&1].id", + "demandDetailId": "Bill.billDetails.billAccountDetails.[&1].demandDetailId" + } + } + } + }, + "isCancelled": "Bill.isCancelled", + "paidBy": "Bill.paidBy", + "mobileNumber": "Bill.mobileNumber", + "payerId": "Bill.payerId", + "isActive": "Bill.isActive", + "additionalDetails": "Bill.additionalDetails", + "payerAddress": "Bill.payerAddress", + "taxAndPayments": "Bill.taxAndPayments", + "auditDetails": "Bill.auditDetails", + "tenantId": "Bill.tenantId", + "payerName": "Bill.payerName", + "id": "Bill.id", + "payerEmail": "Bill.payerEmail" + } + }, + "instrument": { + "bankAccount": "instrument.bankAccount", + "transactionDateInput": "instrument.transactionDateInput", + "amount": "instrument.amount", + "instrumentType": { + "instrumentTypeProperties": "instrument.typeProperties", + "name": "instrument.typeName", + "id": "instrument.typeId" + }, + "instrumentNumber": "instrument.number" + }, + "tenantData": { + "code": "tenantData.code", + "name": "tenantData.name", + "type": "tenantData.type", + "city": { + "name": "tenantData.cityName", + "localName": "tenantData.cityLocalName", + "districtCode": "tenantData.cityDistrictCode", + "districtName": "tenantData.cityDistrictName", + "regionName": "tenantData.cityRegionName", + "ulbGrade": "tenantData.cityUlbGrade", + "code": "tenantData.cityCode" + } + } + } + }, { + "operation": "default", + "spec": { + "domainObject": "{{domainObject}}" + } + } +] diff --git a/dashboard-ingest/src/main/resources/config/transform_pgr_v1.json b/dashboard-ingest/src/main/resources/config/transform_pgr_v1.json new file mode 100644 index 000000000..e1d93fe4c --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_pgr_v1.json @@ -0,0 +1,12 @@ +[ + { + "operation": "shift", + "spec": { + "tenantId": "tenantId", + "serviceRequestId": "serviceRequestId", + "complainCategory": "complainCategory", + "dateOfComplaint": "dateOfComplaint" + + } + } +] diff --git a/dashboard-ingest/src/main/resources/config/transform_pt_v1.json b/dashboard-ingest/src/main/resources/config/transform_pt_v1.json new file mode 100644 index 000000000..aba9d105f --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_pt_v1.json @@ -0,0 +1,10 @@ +[ + { + "operation": "shift", + "spec": { + "tenantId": "tenantId", + "propertyId": "propertyId", + "propertyDetails": "propertyDetails" + } + } +] diff --git a/dashboard-ingest/src/main/resources/config/transform_target_v1.json b/dashboard-ingest/src/main/resources/config/transform_target_v1.json new file mode 100644 index 000000000..da01e1ffe --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_target_v1.json @@ -0,0 +1,20 @@ +[ + { + "operation": "shift", + "spec": { + "*": { + "S. No.": "[&1].snoForMunicipalCorporation", + "Tenant Id": "[&1].tenantIdForMunicipalCorporation", + "ULB Name": "[&1].ulbName", + "Actual Collection (in Lakhs)": "[&1].actualCollectionForMunicipalCorporation", + "Budget Proposed (in Lakhs)": "[&1].budgetProposedForMunicipalCorporation", + "% Actual Collection over Budgeted": "[&1].actualCollectionBudgetedForMunicipalCorporation", + "Financial Year": "[&1].financialYear", + "Timestamp": "[&1].Timestamp", + "Sheet Name": "[&1].businessService" + } + } + } +] + + diff --git a/dashboard-ingest/src/main/resources/config/transform_tl_v1.json b/dashboard-ingest/src/main/resources/config/transform_tl_v1.json new file mode 100644 index 000000000..f838baaec --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_tl_v1.json @@ -0,0 +1,27 @@ +[ + { + "operation": "shift", + "spec": { + "tradelicense": { + "tenantid": "tenantId", + "calculation": "tradeLicense.calculation", + "licensenumber": "tradeLicense.licenseNumber", + "commencementdate": "tradeLicense.commencementDate", + "licensetype": "tradeLicense.licenseType", + "financialyear": "tradeLicense.finYear", + "tradename": "tradeLicense.tradeName", + "applicationnumber": "tradeLicense.applicationNumber", + "accountid": "tradeLicense.accountId", + "applicationdate": "tradeLicense.applicationDate", + "oldpropertyid": "tradeLicense.oldPropertyId", + "action": "tradeLicense.action" + } + } + } +] + + + + + + diff --git a/dashboard-ingest/src/main/resources/config/transform_transaction_v1.json b/dashboard-ingest/src/main/resources/config/transform_transaction_v1.json new file mode 100644 index 000000000..0fd6b03ef --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/transform_transaction_v1.json @@ -0,0 +1,29 @@ + [ + { + "operation": "shift", + "spec": { + "rating": { + "primary": { + "value": "Rating", + "max": "RatingRange" + }, + "*": { + "max": "SecondaryRatings.&1.Range", + "value": "SecondaryRatings.&1.Value", + "$": "SecondaryRatings.&1.Id" + } + } + } + }, + { + "operation": "default", + "spec": { + "Range": 5, + "SecondaryRatings": { + "*": { + "Range": 5 + } + } + } + } + ] diff --git a/dashboard-ingest/src/main/resources/config/validator_transaction_v1.json b/dashboard-ingest/src/main/resources/config/validator_transaction_v1.json new file mode 100644 index 000000000..e451ebda5 --- /dev/null +++ b/dashboard-ingest/src/main/resources/config/validator_transaction_v1.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "transaction", + "description": "Each Transaction has to go through this validation conditions", + "type": "object", + "items": { + "type": "object", + "properties": { + "rating": { + "type": "object", + "items": { + "type": "object", + "properties": { + "primary": { + "type": "object", + "properties": { + "value": { + "type": "integer", + "minimum": 1, + "maximum": 5 + } + } + } + } + } + } + } + }, + "required": [ + "rating" + ] +} \ No newline at end of file diff --git a/dashboard-ingest/src/main/resources/data.sql b/dashboard-ingest/src/main/resources/data.sql new file mode 100644 index 000000000..e8d22e21f --- /dev/null +++ b/dashboard-ingest/src/main/resources/data.sql @@ -0,0 +1,13 @@ +/*DROP TABLE IF EXISTS targetcollection; + +CREATE TABLE targetcollection ( + Serial Number INT PRIMARY KEY, + City Name VARCHAR(250) NOT NULL, + Code VARCHAR(250) NOT NULL, + Budget Proposed(in Lakhs) VARCHAR(250) NOT NULL, + Actual Collection(in Lakhs) VARCHAR(250) NOT NULL + % Actual Collection over Budgeted VARCHAR(250) NOT NULL +); + +INSERT INTO targetcollection (Serial Number, City Name, Code, Budget Proposed(in Lakhs), Actual Collection(in Lakhs),% Actual Collection over Budgeted) VALUES + (?,?,?,?,?,?)*/ \ No newline at end of file diff --git a/dashboard-ingest/src/test/java/com/ingestpipeline/testcases/IngestTestClient.java b/dashboard-ingest/src/test/java/com/ingestpipeline/testcases/IngestTestClient.java new file mode 100644 index 000000000..ea23eb13b --- /dev/null +++ b/dashboard-ingest/src/test/java/com/ingestpipeline/testcases/IngestTestClient.java @@ -0,0 +1,54 @@ +package com.ingestpipeline.testcases; + +import java.util.LinkedHashMap; +import java.util.List; + +import org.springframework.web.client.RestTemplate; + + +public class IngestTestClient { + + public static final String REST_SERVICE_URI = "http://localhost:8081/ingest/api"; + + /* GET */ + @SuppressWarnings("unchecked") + private static void listAllUsers(){ + System.out.println("Testing listAllUsers API-----------"); + + RestTemplate restTemplate = new RestTemplate(); + List> usersMap = restTemplate.getForObject(REST_SERVICE_URI+"/user/", List.class); + + if(usersMap!=null){ + for(LinkedHashMap map : usersMap){ + System.out.println("User : id="+map.get("id")+", Name="+map.get("name")+", Age="+map.get("age")+", Salary="+map.get("salary"));; + } + }else{ + System.out.println("No user exist----------"); + } + } + + /* DELETE */ + private static void deleteUser() { + System.out.println("Testing delete User API----------"); + RestTemplate restTemplate = new RestTemplate(); + restTemplate.delete(REST_SERVICE_URI+"/user/3"); + } + + + /* DELETE */ + private static void deleteAllUsers() { + System.out.println("Testing all delete Users API----------"); + RestTemplate restTemplate = new RestTemplate(); + restTemplate.delete(REST_SERVICE_URI+"/user/"); + } + + public static void main(String args[]){ + listAllUsers(); + listAllUsers(); + listAllUsers(); + //deleteUser(); + listAllUsers(); + //deleteAllUsers(); + listAllUsers(); + } +} \ No newline at end of file From 722788cbccdd4f35c13dc0a325d275bb9b285b02 Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 28 Nov 2019 15:28:20 +0530 Subject: [PATCH 002/152] Updated Dashboard Analytics --- dashboard-analytics/pom.xml | 20 +- .../analytics/ConfigurationLoader.java | 4 +- .../tarento/analytics/constant/Constants.java | 24 +- .../controllers/DashboardController.java | 144 +++- .../analytics/dao/ElasticSearchDao.java | 4 +- .../dao/impl/ElasticSearchDaoImpl.java | 19 +- .../tarento/analytics/dto/AggregateDto.java | 10 + .../analytics/dto/AggregateRequestDto.java | 125 +-- ...tDtoV2.java => AggregateRequestDtoV3.java} | 41 +- .../java/com/tarento/analytics/dto/City.java | 186 ++++ .../com/tarento/analytics/dto/RequestDto.java | 22 + .../{RequestDtoV2.java => RequestDtoV3.java} | 10 +- .../com/tarento/analytics/dto/Tenant.java | 206 +++++ .../com/tarento/analytics/dto/Tenants.java | 66 ++ .../tarento/analytics/dto/Visualization.java | 22 + .../analytics/handler/IResponseHandler.java | 56 +- .../handler/LineChartResponseHandler.java | 62 +- .../handler/MetricChartResponseHandler.java | 41 +- .../PerformanceChartResponeHandler.java | 17 +- .../handler/PieChartResponseHandler.java | 11 +- .../handler/TableChartResponseHandler.java | 32 +- .../analytics/helper/ComputeHelper.java | 17 + .../helper/ComputeHelperFactory.java | 28 + .../helper/TargetPerDateComputeHelper.java | 53 ++ .../analytics/org/service/ClientService.java | 2 +- .../org/service/TarentoServiceImpl.java | 30 +- .../analytics/service/MetadataService.java | 3 + .../analytics/service/QueryService.java | 4 +- .../service/impl/MetadataServiceImpl.java | 138 ++- .../service/impl/QueryServiceImpl.java | 7 +- ...yServiceTemplate.java => RestService.java} | 87 +- .../analytics/service/impl/RetryTemplate.java | 38 + .../analytics/utils/ElasticSearchClient.java | 6 +- .../tarento/analytics/utils/PathRoutes.java | 2 + .../src/main/resources/application.properties | 46 +- .../main/resources/schema/ChartApiConfig.json | 299 +++++-- .../schema/MasterDashboardConfig.json | 808 ++++++++++++++++++ .../resources/schema/RoleDashboardConfig.json | 2 + .../schema/RoleDashboardMappingsConf.json | 27 + 39 files changed, 2374 insertions(+), 345 deletions(-) rename dashboard-analytics/src/main/java/com/tarento/analytics/dto/{AggregateRequestDtoV2.java => AggregateRequestDtoV3.java} (73%) create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/City.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/RequestDto.java rename dashboard-analytics/src/main/java/com/tarento/analytics/dto/{RequestDtoV2.java => RequestDtoV3.java} (63%) create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/Tenant.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/Tenants.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/dto/Visualization.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputeHelper.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/helper/ComputeHelperFactory.java create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/helper/TargetPerDateComputeHelper.java rename dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/{QueryServiceTemplate.java => RestService.java} (52%) create mode 100644 dashboard-analytics/src/main/java/com/tarento/analytics/service/impl/RetryTemplate.java create mode 100644 dashboard-analytics/src/main/resources/schema/MasterDashboardConfig.json create mode 100644 dashboard-analytics/src/main/resources/schema/RoleDashboardMappingsConf.json diff --git a/dashboard-analytics/pom.xml b/dashboard-analytics/pom.xml index 69625fba9..fbeaf9442 100644 --- a/dashboard-analytics/pom.xml +++ b/dashboard-analytics/pom.xml @@ -28,11 +28,6 @@ 3.8.1 test - - mysql - mysql-connector-java - 5.1.21 - org.springframework.kafka spring-kafka @@ -129,9 +124,20 @@ commons-io 2.6 + + + org.json + json + 20180813 + + + com.googlecode.json-simple + json-simple + 1.1 + - +