From a17ebe742af86d646e26fec14e9434e494af269d Mon Sep 17 00:00:00 2001
From: Marian Novotny <46998027+mariannovotny@users.noreply.github.com>
Date: Fri, 4 Feb 2022 13:37:51 +0000
Subject: [PATCH] Siembol release 2.2.0 (#526)
---
CHANGELOG.md | 12 +++++++++++-
alerting/alerting-core/pom.xml | 4 ++--
alerting/alerting-spark/pom.xml | 4 ++--
alerting/alerting-storm/pom.xml | 4 ++--
alerting/pom.xml | 2 +-
config-editor/config-editor-core/pom.xml | 4 ++--
config-editor/config-editor-rest/pom.xml | 18 +++++++++---------
config-editor/config-editor-services/pom.xml | 14 +++++++-------
config-editor/config-editor-sync/pom.xml | 8 ++++----
config-editor/config-editor-ui/package.json | 2 +-
config-editor/pom.xml | 2 +-
deployment/storm-topology-manager/pom.xml | 4 ++--
docs/services/images/header_router_parsing.svg | 1 +
docs/services/images/topic_router_parsing.svg | 1 +
docs/services/siembol_enrichment_service.md | 1 -
docs/services/siembol_parsing_services.md | 18 ++++++++++++++++--
enriching/enriching-core/pom.xml | 6 +++---
enriching/enriching-storm/pom.xml | 4 ++--
enriching/pom.xml | 2 +-
parsing/parsing-app/pom.xml | 6 +++---
parsing/parsing-core/pom.xml | 4 ++--
parsing/parsing-storm/pom.xml | 4 ++--
parsing/pom.xml | 2 +-
pom.xml | 2 +-
responding/pom.xml | 2 +-
responding/responding-core/pom.xml | 6 +++---
responding/responding-stream/pom.xml | 6 +++---
siembol-common/pom.xml | 2 +-
28 files changed, 85 insertions(+), 60 deletions(-)
create mode 100644 docs/services/images/header_router_parsing.svg
create mode 100644 docs/services/images/topic_router_parsing.svg
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fc8789c7c..d24a1088f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [2.2.0] - 2022-02-04
+
+- Reworking kafka writing in Storm applications to use async sending in order to improve performance
+- Deprecating custom batching in kafka writer and use Storm and Kafka batching instead
+- Upgrading Apache Kafka dependency to 3.1.0 and Spring Boot applications to 2.6.3
+- Upgrading Angular to 13.2.0 in Siembol UI
+- Adding management page into Siembol UI including management links and restarting all Siembol Storm applications
+- Adding topic routing parsing and header routing parsing application types
+- Various fixes required for Siembol k8s deployment
+
## [2.1.0] - 2021-11-19
- Supporting autocreation of ZooKeeper nodes in ZooKeeper connector
@@ -44,4 +54,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [1.0.0] - 2021-04-27
-- First public release
\ No newline at end of file
+- First public release
diff --git a/alerting/alerting-core/pom.xml b/alerting/alerting-core/pom.xml
index 706eaa68c..9db9cf186 100644
--- a/alerting/alerting-core/pom.xml
+++ b/alerting/alerting-core/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
alerting
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -35,7 +35,7 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
junit
diff --git a/alerting/alerting-spark/pom.xml b/alerting/alerting-spark/pom.xml
index 0fe242055..38495de0f 100644
--- a/alerting/alerting-spark/pom.xml
+++ b/alerting/alerting-spark/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
alerting
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -23,7 +23,7 @@
uk.co.gresearch.siembol
alerting-core
- 2.2.13-SNAPSHOT
+ 2.2.0
jackson-databind
diff --git a/alerting/alerting-storm/pom.xml b/alerting/alerting-storm/pom.xml
index a638d0af4..44037e840 100644
--- a/alerting/alerting-storm/pom.xml
+++ b/alerting/alerting-storm/pom.xml
@@ -9,7 +9,7 @@
uk.co.gresearch.siembol
alerting
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -51,7 +51,7 @@
uk.co.gresearch.siembol
alerting-core
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
diff --git a/alerting/pom.xml b/alerting/pom.xml
index bf0e53597..8f98472dd 100644
--- a/alerting/pom.xml
+++ b/alerting/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
siembol
- 2.2.13-SNAPSHOT
+ 2.2.0
alerting-core
diff --git a/config-editor/config-editor-core/pom.xml b/config-editor/config-editor-core/pom.xml
index 20e27c8ba..6a72babcf 100644
--- a/config-editor/config-editor-core/pom.xml
+++ b/config-editor/config-editor-core/pom.xml
@@ -9,13 +9,13 @@
uk.co.gresearch.siembol
config-editor
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
org.apache.commons
diff --git a/config-editor/config-editor-rest/pom.xml b/config-editor/config-editor-rest/pom.xml
index 817648274..707bbe291 100644
--- a/config-editor/config-editor-rest/pom.xml
+++ b/config-editor/config-editor-rest/pom.xml
@@ -9,7 +9,7 @@
uk.co.gresearch.siembol
config-editor
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -56,7 +56,7 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
@@ -67,22 +67,22 @@
uk.co.gresearch.siembol
config-editor-core
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
config-editor-services
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
config-editor-sync
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
alerting-core
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
@@ -93,7 +93,7 @@
uk.co.gresearch.siembol
parsing-app
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
@@ -104,7 +104,7 @@
uk.co.gresearch.siembol
enriching-core
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
@@ -115,7 +115,7 @@
uk.co.gresearch.siembol
responding-core
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
diff --git a/config-editor/config-editor-services/pom.xml b/config-editor/config-editor-services/pom.xml
index b708457d1..de31593cd 100644
--- a/config-editor/config-editor-services/pom.xml
+++ b/config-editor/config-editor-services/pom.xml
@@ -10,7 +10,7 @@
uk.co.gresearch.siembol
config-editor
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -41,32 +41,32 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
config-editor-core
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
alerting-core
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
parsing-app
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
enriching-core
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
responding-core
- 2.2.13-SNAPSHOT
+ 2.2.0
junit
diff --git a/config-editor/config-editor-sync/pom.xml b/config-editor/config-editor-sync/pom.xml
index 65b51dd65..bad6bf9a9 100644
--- a/config-editor/config-editor-sync/pom.xml
+++ b/config-editor/config-editor-sync/pom.xml
@@ -9,7 +9,7 @@
uk.co.gresearch.siembol
config-editor
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -20,17 +20,17 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
config-editor-core
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
parsing-app
- 2.2.13-SNAPSHOT
+ 2.2.0
provided
diff --git a/config-editor/config-editor-ui/package.json b/config-editor/config-editor-ui/package.json
index fabc2e9d9..40d08c539 100644
--- a/config-editor/config-editor-ui/package.json
+++ b/config-editor/config-editor-ui/package.json
@@ -1,6 +1,6 @@
{
"name": "rule-editor.ui",
- "version": "2.2.8-dev",
+ "version": "2.2.0",
"license": "MIT",
"scripts": {
"ng": "ng",
diff --git a/config-editor/pom.xml b/config-editor/pom.xml
index 1e14a2b0f..2771bb7fe 100644
--- a/config-editor/pom.xml
+++ b/config-editor/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
siembol
- 2.2.13-SNAPSHOT
+ 2.2.0
config-editor-core
diff --git a/deployment/storm-topology-manager/pom.xml b/deployment/storm-topology-manager/pom.xml
index 175b8ed1f..99c6c5936 100644
--- a/deployment/storm-topology-manager/pom.xml
+++ b/deployment/storm-topology-manager/pom.xml
@@ -9,7 +9,7 @@
uk.co.gresearch.siembol
siembol
- 2.2.13-SNAPSHOT
+ 2.2.0
../../pom.xml
@@ -43,7 +43,7 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
diff --git a/docs/services/images/header_router_parsing.svg b/docs/services/images/header_router_parsing.svg
new file mode 100644
index 000000000..9b8d169e5
--- /dev/null
+++ b/docs/services/images/header_router_parsing.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/services/images/topic_router_parsing.svg b/docs/services/images/topic_router_parsing.svg
new file mode 100644
index 000000000..a32e2a965
--- /dev/null
+++ b/docs/services/images/topic_router_parsing.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/services/siembol_enrichment_service.md b/docs/services/siembol_enrichment_service.md
index 98f66683a..f9307603a 100644
--- a/docs/services/siembol_enrichment_service.md
+++ b/docs/services/siembol_enrichment_service.md
@@ -67,7 +67,6 @@ Note: you can only enrich from one table per rule. If you want to enrich the sam
- `zk.url` - Zookeeper servers url. Multiple servers are separated by comma
- `zk.path` - Path to a zookeeper node
- `kafka.batch.writer.attributes` - Kafka batch writer attributes for producing output messages
- - `batch.size` - The max size of batch used for producing messages
- `producer.properties` - Defines kafka producer properties, see [https://kafka.apache.org/0102/documentation.html#producerconfigs](https://kafka.apache.org/0102/documentation.html#producerconfigs)
- `storm.attributes` - Storm attributes for the enrichment topology
- `bootstrap.servers` - Kafka brokers servers url. Multiple servers are separated by comma
diff --git a/docs/services/siembol_parsing_services.md b/docs/services/siembol_parsing_services.md
index 3a1f05cb1..919784ea8 100644
--- a/docs/services/siembol_parsing_services.md
+++ b/docs/services/siembol_parsing_services.md
@@ -107,7 +107,7 @@ Parsers are integrated in a stream application (storm topology) that combines on
- `parsing_app_autho` - The author of the parsing application
- `parsing_app_description`- Description of the parsing application
- `parsing_app_settings` - Parsing application settings
- - `parsing_app_type`- The type of the parsing application - `router_parsing` or `single_parser`
+ - `parsing_app_type`- The type of the parsing application - `single_parser`, `router_parsing`, `topic_routing_parsing` or `header_routing_parsing`
- `input_topics` - The kafka topics for reading messages for parsing
- `error_topic`- The kafka topic for publishing error messages
- `input_parallelism` - The number of parallel executors for reading messages from the input kafka topics
@@ -130,6 +130,21 @@ The application integrates multiple parsers. First, the router parser parses the
- `parsers` - The list of parsers for further parsing
- `routing_field_pattern` - The pattern for selecting the parser
- `parser_properties` - The properties of the selected parser with `parser_name` and `output_topic`
+### Topic routing parsing
+
+The application integrates multiple parsers and reads logs from multiple topics. The parser is selected based on the topic name on which the log was received.
+- `default_parser` - The parser that should be used if no other parsers is selected with `parser_name` and `output_topic`
+- `parsers` - The list of parsers for further parsing
+ - `topic_name` - The name of the topic for selecting the parser
+ - `parser_properties` - The properties of the selected parser with `parser_name` and `output_topic`
+### Header routing parsing
+
+The application integrates multiple parsers and uses a kafka message header for routing. The parser is selected based on the dedicated header value.
+- `default_parser` - The parser that should be used if no other parsers is selected with `parser_name` and `output_topic`
+- `header_name` - The name of the header used for routing
+- `parsers` - The list of parsers for further parsing
+ - `source_header_value` - The value in the header for selecting the parser
+ - `parser_properties` - The properties of the selected parser with `parser_name` and `output_topic`
## Admin Config
- `topology.name.prefix` - The prefix that will be used to create a topology name using the application name, by default `parsing`
- `client.id.prefix` - The prefix that will be used to create a kafka producer client id using the application name
@@ -138,7 +153,6 @@ The application integrates multiple parsers. First, the router parser parses the
- `zk.url` - Zookeeper servers url. Multiple servers are separated by a comma
- `zk.path` - Path to a zookeeper node
- `kafka.batch.writer.attributes` - Global settings for the kafka batch writer used if they are not overridden
- - `batch.size` - The max size of batch used for producing messages
- `producer.properties` - Defines kafka producer properties, see [https://kafka.apache.org/0102/documentation.html#producerconfigs](https://kafka.apache.org/0102/documentation.html#producerconfigs)
- `storm.attributes` - Global settings for storm attributes used if they are not overridden
- `bootstrap.servers` - Kafka brokers servers url. Multiple servers are separated by a comma
diff --git a/enriching/enriching-core/pom.xml b/enriching/enriching-core/pom.xml
index 4d547b8fa..2ade64f85 100644
--- a/enriching/enriching-core/pom.xml
+++ b/enriching/enriching-core/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
enriching
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -35,12 +35,12 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
alerting-core
- 2.2.13-SNAPSHOT
+ 2.2.0
junit
diff --git a/enriching/enriching-storm/pom.xml b/enriching/enriching-storm/pom.xml
index 6a6a8f021..4ef06f961 100644
--- a/enriching/enriching-storm/pom.xml
+++ b/enriching/enriching-storm/pom.xml
@@ -9,7 +9,7 @@
uk.co.gresearch.siembol
enriching
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -75,7 +75,7 @@
uk.co.gresearch.siembol
enriching-core
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
diff --git a/enriching/pom.xml b/enriching/pom.xml
index 183184bab..3ba782106 100644
--- a/enriching/pom.xml
+++ b/enriching/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
siembol
- 2.2.13-SNAPSHOT
+ 2.2.0
enriching-core
diff --git a/parsing/parsing-app/pom.xml b/parsing/parsing-app/pom.xml
index 247dcc009..3f8aa8329 100644
--- a/parsing/parsing-app/pom.xml
+++ b/parsing/parsing-app/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
parsing
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -39,12 +39,12 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
parsing-core
- 2.2.13-SNAPSHOT
+ 2.2.0
junit
diff --git a/parsing/parsing-core/pom.xml b/parsing/parsing-core/pom.xml
index 0449203f2..13577243b 100644
--- a/parsing/parsing-core/pom.xml
+++ b/parsing/parsing-core/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
parsing
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -45,7 +45,7 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
joda-time
diff --git a/parsing/parsing-storm/pom.xml b/parsing/parsing-storm/pom.xml
index 22fb6458a..f73442466 100644
--- a/parsing/parsing-storm/pom.xml
+++ b/parsing/parsing-storm/pom.xml
@@ -9,7 +9,7 @@
uk.co.gresearch.siembol
parsing
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -75,7 +75,7 @@
uk.co.gresearch.siembol
parsing-app
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
diff --git a/parsing/pom.xml b/parsing/pom.xml
index be0b4d0e5..6d5441a87 100644
--- a/parsing/pom.xml
+++ b/parsing/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
siembol
- 2.2.13-SNAPSHOT
+ 2.2.0
parsing-core
diff --git a/pom.xml b/pom.xml
index a663a35e6..60f092ea5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
uk.co.gresearch.siembol
siembol
siembol
- 2.2.13-SNAPSHOT
+ 2.2.0
A scalable, advanced security analytics framework based on open-source big data technologies.
2019
https://siembol.io/
diff --git a/responding/pom.xml b/responding/pom.xml
index 3272869b0..36998a972 100644
--- a/responding/pom.xml
+++ b/responding/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
siembol
- 2.2.13-SNAPSHOT
+ 2.2.0
responding-core
diff --git a/responding/responding-core/pom.xml b/responding/responding-core/pom.xml
index f26596f95..4b92baab4 100644
--- a/responding/responding-core/pom.xml
+++ b/responding/responding-core/pom.xml
@@ -11,7 +11,7 @@
uk.co.gresearch.siembol
responding
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -35,12 +35,12 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
uk.co.gresearch.siembol
alerting-core
- 2.2.13-SNAPSHOT
+ 2.2.0
com.jayway.jsonpath
diff --git a/responding/responding-stream/pom.xml b/responding/responding-stream/pom.xml
index 7d7aa4fa4..d701090a3 100644
--- a/responding/responding-stream/pom.xml
+++ b/responding/responding-stream/pom.xml
@@ -9,7 +9,7 @@
uk.co.gresearch.siembol
responding
- 2.2.13-SNAPSHOT
+ 2.2.0
@@ -51,7 +51,7 @@
uk.co.gresearch.siembol
siembol-common
- 2.2.13-SNAPSHOT
+ 2.2.0
org.slf4j
@@ -62,7 +62,7 @@
uk.co.gresearch.siembol
responding-core
- 2.2.13-SNAPSHOT
+ 2.2.0
org.apache.kafka
diff --git a/siembol-common/pom.xml b/siembol-common/pom.xml
index 19e2b407d..f78a450b1 100644
--- a/siembol-common/pom.xml
+++ b/siembol-common/pom.xml
@@ -9,7 +9,7 @@
uk.co.gresearch.siembol
siembol
- 2.2.13-SNAPSHOT
+ 2.2.0