[Cisco Secure Email Gateway] Fix for grok if hostname appears before …

…category.name in Cisco secure email gateway package #9160)
elastic · Mar 13, 2024 · e5b7193 · e5b7193
1 parent f3212c0
commit e5b7193
Show file tree

Hide file tree

Showing 5 changed files with 115 additions and 3 deletions.
diff --git a/packages/cisco_secure_email_gateway/changelog.yml b/packages/cisco_secure_email_gateway/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.22.2"
+  changes:
+    - description: Fix grok if hostname appears before category.name
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9160
 - version: "1.22.1"
   changes:
     - description: Changed owners

diff --git a/...ecure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log b/...ecure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log
@@ -12,6 +12,7 @@
 <14>Jul 04 06:21:54 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=E7DEF468022C4EB09683-9A331A42E1F7 ESAMID=54376810 ESAICID=43587623 endTime=Tue Jul  4 06:21:54 2023 ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.144 ESAAttachmentDetails={'meeting.ics': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '17ae79446b4ec3baf161704831970aac49457d32935c6383c2a45aed136a99df'}, 'BodyScanner': {}}} ESAFriendlyFrom=River <river@this.example.com> ESAGMVerdict=NEGATIVE startTime=Tue Jul  4 16:12:44 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=river@this.example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=New Zealand ESAMFVerdict=MATCH act=DQ ESAFinalActionDetails=Message held temporarily in Delay Quarantine cs4Label=ExternalMsgID cs4='<2403354681.734500.1688449973515.mail.lion@example.com>' ESAMsgSize=18675 ESAOFVerdict=NEGATIVE duser=smith@example.com ESAHeloDomain=vm-lion.dmz ESAHeloIP=89.160.20.128 cfp1Label=SBRSScore cfp1=None ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'river@this.example.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@vm-lion.dmz'}} sourceHostName=company.example.com sourceAddress=89.160.20.128 msg='Accept: Cisco - SOLUTIONS' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=FA_PENDING ESAASVerdict=NEGATIVE ESAAVVerdict=NOT_EVALUATED ESACFVerdict=MATCH
 <14>Jul 04 06:21:54 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=30543A3AB9E54FA8AAC1-FB812C95028D ESAMID=238746 ESAICID=435897324 ESADCID=34809573 endTime=Tue Jul  4 06:21:54 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.142 ESAFriendlyFrom=Will <irobot@example.com> ESAGMVerdict=NEGATIVE startTime=Tue Jul  4 15:14:29 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=irobot@example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=United States ESAMFVerdict=MATCH act=QUARANTINED ESAFinalActionDetails=To SPAM cs4Label=ExternalMsgID cs4='<MDlhMDg0MjY0NmE2OWFkNTZhMzA2NDA0MDVkZWNlZWVlYzI3MjMyYmI5YWJlNDMxM2UxOGVjZTBiNGZmOGZmYSAgLQo@hotmail.com>' ESAMsgSize=12312 ESAOFVerdict=NEGATIVE duser=alfombra@example.com ESAHeloDomain=mail-q6by9-a42.google.com ESAHeloIP=81.2.69.192 cfp1Label=SBRSScore cfp1=2.7 ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'irobot@example.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@mail-q6by9-a42.google.com'}} sourceHostName=mail-q6by9-a42.google.com ESASenderGroup=ACCEPTLIST sourceAddress=81.2.69.192 msg='IE : Crayons' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=MATCH
 <14>Jul 04 06:21:54 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=4FEF3A4372664BCCB404-20EE1767D434 ESAMID=786324 ESAICID=35635425 ESADCID=970897 endTime=Tue Jul  4 06:21:54 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.144 ESAFriendlyFrom=Beaches <playas@example.com> ESAGMVerdict=NEGATIVE startTime=Tue Jul  4 14:42:34 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=westinghouse-thoreau\\=example.com@example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=NZ ESAMFVerdict=MATCH act=QUARANTINED ESAFinalActionDetails=To SPAM cs4Label=ExternalMsgID cs4='<490b2a15fa4742331779cdaa4e@example.com>' ESAMsgSize=20668 ESAOFVerdict=NEGATIVE duser=thoreau@example.com ESAHeloDomain=example.com ESAHeloIP=89.160.20.112 ESAReplyTo=lane@example.com cfp1Label=SBRSScore cfp1=None ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'westinghouse-thoreau=example.com@example.com'}, 'helo': {'result': 'Pass', 'sender': 'postmaster@example.com'}} sourceHostName=example.com sourceAddress=89.160.20.112 msg=\"Totally not suspicious email subject\" ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NO_MATCH
+<14>Jul 04 06:21:54 test.hostname.co consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=4FEF3A4372664BCCB404-20EE1767D434 ESAMID=786324 ESAICID=35635425 ESADCID=970897 endTime=Tue Jul  4 06:21:54 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.144 ESAFriendlyFrom=Beaches <playas@example.com> ESAGMVerdict=NEGATIVE startTime=Tue Jul  4 14:42:34 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=westinghouse-thoreau\\=example.com@example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=NZ ESAMFVerdict=MATCH act=QUARANTINED ESAFinalActionDetails=To SPAM cs4Label=ExternalMsgID cs4='<490b2a15fa4742331779cdaa4e@example.com>' ESAMsgSize=20668 ESAOFVerdict=NEGATIVE duser=thoreau@example.com ESAHeloDomain=example.com ESAHeloIP=89.160.20.112 ESAReplyTo=lane@example.com cfp1Label=SBRSScore cfp1=None ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'westinghouse-thoreau=example.com@example.com'}, 'helo': {'result': 'Pass', 'sender': 'postmaster@example.com'}} sourceHostName=example.com sourceAddress=89.160.20.112 msg=\"Totally not suspicious email subject\" ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NO_MATCH
 Sep 16 16:56:06 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|15.0.0-104|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=4235746AE34E6DD44EB8-CE101C994AA5 ESAMID=2348764 ESAICID=435786234 ESADCID=23876232 end=Tue Sep 16 16:56:06 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.144 ESAFriendlyFrom=Bouncy Castle - No Reply ESAGMVerdict=NEGATIVE start=Tue Sep 16 16:56:04 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=0101016d24913de4eb-28ea32e6-4dad-4943-b6f7-ef0dddbb875d-000000@bounce.com cs1Label=MailPolicy cs1=Exception_support cs2Label=SenderCountry cs2=United States ESAMFVerdict=MATCH act=DELIVERED cs4Label=ExternalMsgID cs4='' ESAMsgSize=3518 ESAOFVerdict=NOT_EVALUATED duser=support@example.com ESAHeloDomain=smtp-out.us-west-1.amazon.com ESAHeloIP=216.160.83.56 cfp1Label=SBRSScore cfp1=3.5 ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Favorable ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': '0101016d24913de4eb-28ea32e6-4dad-4943-b6f7-ef0dddbb875d-000000@bounce.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@smtp-out.us-west-1.amazon.com'}} shost=smtp-out.us-west-1.amazon.com ESASenderGroup=ACCEPTLIST src=216.160.83.56 msg='Alert - wireless - APs came up' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESATLSOutCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSOutConnStatus=Success ESATLSOutProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=NEGATIVE ESAAVVerdict=NEGATIVE ESACFVerdict=NOT_EVALUATED
 Sep 16 16:56:06 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|15.0.0-104|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=4235746AE34E6DD44EB8-CE101C994AA5 ESAMID=2348764 ESAICID=435786234 ESADCID=23876232 ESAAMPVerdict=SKIPPED ESAAVVerdict=NEGATIVE ESAFriendlyFrom=Bouncy Castle <bouncy@castle.com> duser=support@example.com ESAHeloDomain=smtp-out.us-west-1.amazon.com ESAHeloIP=216.160.83.56 sourceHostName=example.com ESASenderGroup=UNKNOWNLIST sourceAddress=89.160.20.112 msg='Automatic reply: Your weekend reading from Bouncy Castle' ESAURLDetails={'https://example.com': {'WbrsScore': 3.0}, 'http://www.example.com': {'WbrsScore': 8.9000000000000004}, 'http://schemas.microsoft.com/office': {'WbrsScore': 9.0}}
 Sep 17 16:56:06 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|15.0.0-104|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=4235746AE34E6DD44EB8-CE101C994AA5 ESAMID=2348764 ESAICID=435786234 ESADCID=23876232 ESAAMPVerdict=NOT_EVALUATED ESAAVVerdict=NEGATIVE ESAFriendlyFrom="Castle. B (Bouncy Castle)" <bouncy@castle.com> suser=bouncy@castle.com duser=support@example.com ESAHeloDomain=smtp-out.us-west-1.amazon.com ESAHeloIP=216.160.83.56 sourceHostName=unknown ESASenderGroup=RELAYLIST sourceAddress=89.160.20.112 msg='Testing'
diff --git a/...teway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log-expected.json b/...teway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log-expected.json
@@ -1515,6 +1515,112 @@
                 "preserve_original_event"
             ]
         },
+        {
+            "@timestamp": "2024-07-04T06:21:54.000Z",
+            "cisco_secure_email_gateway": {
+                "log": {
+                    "act": "QUARANTINED",
+                    "appliance": {
+                        "product": "C100V Email Security Virtual Appliance",
+                        "vendor": "Cisco",
+                        "version": "14.3.0-032"
+                    },
+                    "category": {
+                        "name": "consolidated_event"
+                    },
+                    "cef_format_version": "0",
+                    "cfp1_label": "SBRSScore",
+                    "cs1": "DEFAULT",
+                    "cs1_label": "MailPolicy",
+                    "cs2": "NZ",
+                    "cs2_label": "SenderCountry",
+                    "cs3": "N/A",
+                    "cs3_label": "SDRThreatCategory",
+                    "cs4": "490b2a15fa4742331779cdaa4e@example.com",
+                    "cs4_label": "ExternalMsgID",
+                    "cs6": "Neutral",
+                    "cs6_label": "SDRRepScore",
+                    "data": {
+                        "ip": "81.2.69.144"
+                    },
+                    "device_direction": "incoming",
+                    "esa": {
+                        "delivery_connection_id": "970897",
+                        "dkim_verdict": "pass",
+                        "dlp_verdict": "NOT_EVALUATED",
+                        "final_action_details": "To SPAM",
+                        "friendly_from": "Beaches <playas@example.com>",
+                        "graymail_verdict": "NEGATIVE",
+                        "helo": {
+                            "domain": "example.com",
+                            "ip": "89.160.20.112"
+                        },
+                        "injection_connection_id": "35635425",
+                        "mail_flow_policy": "ACCEPT",
+                        "mf_verdict": "MATCH",
+                        "msg_size": 20668,
+                        "outbreak_filter_verdict": "NEGATIVE",
+                        "reply_to": "lane@example.com",
+                        "sdr_consolidated_domain_age": "30 days (or greater)",
+                        "spf_verdict": "{'mailfrom': {'result': 'Pass', 'sender': 'westinghouse-thoreau=example.com@example.com'}, 'helo': {'result': 'Pass', 'sender': 'postmaster@example.com'}}"
+                    },
+                    "event": {
+                        "name": "Consolidated Log Event"
+                    },
+                    "event_class_id": "ESA_CONSOLIDATED_LOG_EVENT",
+                    "host": "test.hostname.co",
+                    "listener": {
+                        "name": "IncomingMail"
+                    },
+                    "message": "\\\"Totally not suspicious email subject\\\" ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NO_MATCH"
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "email": {
+                "from": {
+                    "address": [
+                        "westinghouse-thoreau\\\\=example.com@example.com"
+                    ]
+                },
+                "message_id": "786324",
+                "to": {
+                    "address": [
+                        "thoreau@example.com"
+                    ]
+                }
+            },
+            "event": {
+                "end": "2023-07-04T06:21:54.000Z",
+                "kind": "event",
+                "original": "<14>Jul 04 06:21:54 test.hostname.co consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=4FEF3A4372664BCCB404-20EE1767D434 ESAMID=786324 ESAICID=35635425 ESADCID=970897 endTime=Tue Jul  4 06:21:54 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.144 ESAFriendlyFrom=Beaches <playas@example.com> ESAGMVerdict=NEGATIVE startTime=Tue Jul  4 14:42:34 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=westinghouse-thoreau\\\\=example.com@example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=NZ ESAMFVerdict=MATCH act=QUARANTINED ESAFinalActionDetails=To SPAM cs4Label=ExternalMsgID cs4='<490b2a15fa4742331779cdaa4e@example.com>' ESAMsgSize=20668 ESAOFVerdict=NEGATIVE duser=thoreau@example.com ESAHeloDomain=example.com ESAHeloIP=89.160.20.112 ESAReplyTo=lane@example.com cfp1Label=SBRSScore cfp1=None ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'westinghouse-thoreau=example.com@example.com'}, 'helo': {'result': 'Pass', 'sender': 'postmaster@example.com'}} sourceHostName=example.com sourceAddress=89.160.20.112 msg=\\\"Totally not suspicious email subject\\\" ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NO_MATCH",
+                "severity": "5",
+                "start": "2023-07-04T14:42:34.000Z",
+                "timezone": "UTC"
+            },
+            "host": {
+                "id": "4FEF3A4372664BCCB404-20EE1767D434"
+            },
+            "log": {
+                "syslog": {
+                    "priority": 14
+                }
+            },
+            "related": {
+                "ip": [
+                    "89.160.20.112",
+                    "81.2.69.144"
+                ]
+            },
+            "source": {
+                "domain": "example.com",
+                "ip": "89.160.20.112"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
         {
             "@timestamp": "2024-09-16T16:56:06.000Z",
             "cisco_secure_email_gateway": {

diff --git a/...ages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/...ages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -21,8 +21,8 @@ processors:
   - grok:
       field: event.original
       patterns:
-        - '^(?:<%{NUMBER:log.syslog.priority:long}>)?%{SYSLOGTIMESTAMP:_tmp.timestamp} %{DATA:cisco_secure_email_gateway.log.category.name}: %{WORD:log.level}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
-        - '^(?:<%{NUMBER:log.syslog.priority:long}>)?%{SYSLOGTIMESTAMP:_tmp.timestamp} %{DATA:cisco_secure_email_gateway.log.category.name}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
+        - '^(?:<%{NUMBER:log.syslog.priority:long}>)?%{SYSLOGTIMESTAMP:_tmp.timestamp} (?:%{HOSTNAME:cisco_secure_email_gateway.log.host} )?%{DATA:cisco_secure_email_gateway.log.category.name}: %{WORD:log.level}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
+        - '^(?:<%{NUMBER:log.syslog.priority:long}>)?%{SYSLOGTIMESTAMP:_tmp.timestamp} (?:%{HOSTNAME:cisco_secure_email_gateway.log.host} )?%{DATA:cisco_secure_email_gateway.log.category.name}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
         - '^%{DATA:_tmp.timestamp} %{WORD:log.level}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
         - '^%{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
   - trim:

diff --git a/packages/cisco_secure_email_gateway/manifest.yml b/packages/cisco_secure_email_gateway/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: cisco_secure_email_gateway
 title: Cisco Secure Email Gateway
-version: "1.22.1"
+version: "1.22.2"
 description: Collect logs from Cisco Secure Email Gateway with Elastic Agent.
 type: integration
 categories: