feat: yara support (#638)

* feat(api): malware rule name to support yara * feat: yara support * fix: typo in debug message * update temp file path * fix: yara rule metadata parser * update comments * fix: field separator * minor fixes * deploy yara-rule-server with docker compose * feat: add yara config * updated aws installer * updated gcp installer * allow acces to yara server * docs: update * fix: aws installer * fix aws instaler * add yara-rule-server into azure installer * fix lint errors * update doc * fix: yara-rule-server config name * fix missing header * fix makefile * add yara to malware scanner list * installer: use 9993 port for yara rule server * review * fix lint * move pointerToStringOrNil to utils * review * update vmclarity-tool-base * remove debug mesage * fix merge failure * fix select * fix error wrapping * update docker compose file for fix e2e test * update installers * updates based on review * add ruleName to MalwareKey * fix: ruleName in malwareKey and select * fix(install): missing new line * use ValueOrZero instead of StringPointerValOrEmpty
openclarity · Sep 19, 2023 · 32adb9d · 32adb9d
1 parent 7263406
commit 32adb9d
Show file tree

Hide file tree

Showing 39 changed files with 1,032 additions and 179 deletions.
diff --git a/Dockerfile.cli b/Dockerfile.cli
@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.2
-ARG VMCLARITY_TOOLS_BASE=ghcr.io/openclarity/vmclarity-tools-base:v0.3.0@sha256:c592419f5f3f184909363e080b01aea5deef63374b856aed74334afa3bdef793
+ARG VMCLARITY_TOOLS_BASE=ghcr.io/openclarity/vmclarity-tools-base:v0.4.0@sha256:8431af10930dddadaf7365e8610ac8f8f62dd4be01928dba6bc16d50f152a12b
 FROM --platform=$BUILDPLATFORM golang:1.21.1-alpine AS builder
 
 RUN apk add --update --no-cache ca-certificates git

diff --git a/api/models/families.go b/api/models/families.go
@@ -88,7 +88,7 @@ func (c *MalwareConfig) GetScannersList() []string {
 		return *c.Scanners
 	}
 
-	return []string{"clam"}
+	return []string{"clam", "yara"}
 }
 
 func (c *ExploitsConfig) IsEnabled() bool {

diff --git a/api/models/models.gen.go b/api/models/models.gen.go
diff --git a/api/openapi.yaml b/api/openapi.yaml
@@ -2482,6 +2482,8 @@ components:
           type: string
         malwareType:
           $ref: '#/components/schemas/MalwareType'
+        ruleName:
+          type: string
         path:
           type: string
           description: Path of the file that contains malware

diff --git a/api/server/server.gen.go b/api/server/server.gen.go
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -8,13 +8,15 @@
 | `SCANNER_CONTAINER_IMAGE`                 |           |         |                                              |
 | `GITLEAKS_BINARY_PATH`                    |           |         |                                              |
 | `CLAM_BINARY_PATHCLAM_BINARY_PATH`        |           |         |                                              |
+| `YARA_BINARY_PATH`                        |           |         |                                              |
 | `FRESHCLAM_BINARY_PATH`                   |           |         |                                              |
 | `ALTERNATIVE_FRESHCLAM_MIRROR_URL`        |           |         |                                              |
 | `LYNIS_INSTALL_PATH`                      |           |         |                                              |
 | `SCANNER_VMCLARITY_BACKEND_ADDRESS`       |           |         |                                              |
 | `EXPLOIT_DB_ADDRESS`                      |           |         |                                              |
 | `TRIVY_SERVER_ADDRESS`                    |           |         |                                              |
 | `TRIVY_SERVER_TIMEOUT`                    |           |         |                                              |
+| `YARA_RULE_SERVER_ADDRESS`                |           |         |                                              |
 | `GRYPE_SERVER_ADDRESS`                    |           |         |                                              |
 | `GRYPE_SERVER_TIMEOUT`                    |           |         |                                              |
 | `CHKROOTKIT_BINARY_PATH`                  |           |         |                                              |
@@ -23,10 +25,10 @@
 | `SCAN_POLLING_INTERVAL`                   |           |         |                                              |
 | `SCAN_RECONCILE_TIMEOUT`                  |           |         |                                              |
 | `SCAN_TIMEOUT`                            |           |         |                                              |
-| `ASSET_SCAN_POLLING_INTERVAL`            |           |         |                                              |
-| `ASSET_SCAN_RECONCILE_TIMEOUT`           |           |         |                                              |
-| `ASSET_SCAN_PROCESSOR_POLLING_INTERVAL`  |           |         |                                              |
-| `ASSET_SCAN_PROCESSOR_RECONCILE_TIMEOUT` |           |         |                                              |
+| `ASSET_SCAN_POLLING_INTERVAL`             |           |         |                                              |
+| `ASSET_SCAN_RECONCILE_TIMEOUT`            |           |         |                                              |
+| `ASSET_SCAN_PROCESSOR_POLLING_INTERVAL`   |           |         |                                              |
+| `ASSET_SCAN_PROCESSOR_RECONCILE_TIMEOUT`  |           |         |                                              |
 | `DISCOVERY_INTERVAL`                      |           |         |                                              |
 | `CONTROLLER_STARTUP_DELAY`                |           |         |                                              |
 | `PROVIDER`                                | **yes**   | `aws`   | Provider used for Asset discovery and scans |

diff --git a/installation/aws/VmClarity.cfn b/installation/aws/VmClarity.cfn
@@ -227,6 +227,10 @@ Resources:
                   # Create directory for trivy server
                   /usr/bin/mkdir -p /opt/trivy-server
 
+                  # Create directory required for yara-rule-server
+                  /usr/bin/mkdir -p /opt/yara-rule-server
+                  /usr/bin/chown -R 1000:1000 /opt/yara-rule-server
+
                   # Enable and start/restart VMClarity backend
                   systemctl enable vmclarity.service
                   systemctl restart vmclarity.service
@@ -382,6 +386,27 @@ Resources:
                           restart_policy:
                             condition: on-failure
 
+                      yara-rule-server:
+                        image: ${YaraRuleServerContainerImage}
+                        command:
+                          - run
+                        ports:
+                          - "9993:8080"
+                        configs:
+                          - source: yara_rule_server_config
+                            target: /etc/yara-rule-server/config.yaml
+                        volumes:
+                          - type: bind
+                            source: /opt/yara-rule-server
+                            target: /var/lib/yara-rule-server
+                        logging:
+                          driver: journald
+                        deploy:
+                          mode: replicated
+                          replicas: 1
+                          restart_policy:
+                            condition: on-failure
+
                       postgresql:
                         image: ${PostgresqlContainerImage}
                         env_file: ./postgres.env
@@ -410,6 +435,8 @@ Resources:
                         file: ./gateway.conf
                       swagger_config:
                         file: ./swagger-config.json
+                      yara_rule_server_config:
+                        file: ./yara-rule-server.yaml
 
                   - APIServerContainerImage: !If [ APIServerContainerImageOverridden, !Ref APIServerContainerImageOverride, "ghcr.io/openclarity/vmclarity-apiserver:latest" ]
                     OrchestratorContainerImage: !If [ OrchestratorContainerImageOverridden, !Ref OrchestratorContainerImageOverride, "ghcr.io/openclarity/vmclarity-orchestrator:latest" ]
@@ -418,6 +445,7 @@ Resources:
                     ExploitDBServerContainerImage: !If [ExploitDBServerContainerImageOverridden, !Ref ExploitDBServerContainerImageOverride, "ghcr.io/openclarity/exploit-db-server:v0.2.3"]
                     TrivyServerContainerImage: !If [TrivyServerContainerImageOverridden, !Ref TrivyServerContainerImageOverride, "docker.io/aquasec/trivy:0.41.0"]
                     GrypeServerContainerImage: !If [GrypeServerContainerImageOverridden, !Ref GrypeServerContainerImageOverride, "ghcr.io/openclarity/grype-server:v0.5.0"]
+                    YaraRuleServerContainerImage: !If [YaraRuleServerContainerImageOverridden, !Ref YaraRuleServerContainerImageOverride, "ghcr.io/openclarity/yara-rule-server:v0.1.0"]
                     FreshclamMirrorContainerImage: !If [FreshclamMirrorContainerImageOverridden, !Ref FreshclamMirrorContainerImageOverride, "ghcr.io/openclarity/freshclam-mirror:v0.2.0"]
                     PostgresqlContainerImage: !If [PostgresqlContainerImageOverridden, !Ref PostgresqlContainerImageOverride, "bitnami/postgresql:12.14.0-debian-11-r28"]
 
@@ -463,6 +491,8 @@ Resources:
                     GRYPE_SERVER_ADDRESS=__CONTROLPLANE_HOST__:9991
                     # FreshClam mirror URL
                     ALTERNATIVE_FRESHCLAM_MIRROR_URL=http://__CONTROLPLANE_HOST__:1000/clamav
+                    # Yara rule server address
+                    YARA_RULE_SERVER_ADDRESS=http://__CONTROLPLANE_HOST__:9993
                     # Resource cleanup policy
                     DELETE_JOB_POLICY=${AssetScanDeletePolicy}
                     # Provider to use
@@ -508,6 +538,19 @@ Resources:
                 # COMPOSE_PROFILES=
               mode: "000644"
 
+            "/etc/vmclarity/yara-rule-server.yaml":
+              content: |
+                enable_json_log: true
+                rule_update_schedule: "0 0 * * *"
+                rule_sources:
+                  - name: "base"
+                    url: "https://github.com/Yara-Rules/rules/archive/refs/heads/master.zip"
+                    exclude_regex: ".*index.*.yar|.*/utils/.*|.*/deprecated/.*|.*index_.*|.*MALW_AZORULT.yar"
+                  - name: "magic"
+                    url: "https://github.com/securitymagic/yara/archive/refs/heads/main.zip"
+                    exclude_regex: ".*index.*.yar"
+              mode: "000644"
+
             "/etc/vmclarity/postgres.env":
               content:
                 Fn::Sub: |
@@ -674,6 +717,17 @@ Resources:
       FromPort: 1000
       ToPort: 1000
       SourceSecurityGroupId: !Ref VmClarityScannerSecurityGroup
+  # Allow the Scanner VMs in the VmClarityScannerSecurityGroup to access the
+  # Yara Rule Server on port 9993 by adding an ingress rule to the
+  # VmClarityServerSecurityGroup.
+  VmClarityServerSecurityGroupScannerIngressToYaraRuleServer:
+    Type: "AWS::EC2::SecurityGroupIngress"
+    Properties:
+      GroupId: !Ref VmClarityServerSecurityGroup
+      IpProtocol: tcp
+      FromPort: 9993
+      ToPort: 9993
+      SourceSecurityGroupId: !Ref VmClarityScannerSecurityGroup
 
   # Create an Internet Gateway to allow VMClarityServer to talk to the internet
   # and the internet to talk to it for SSH/HTTP.
@@ -1077,6 +1131,12 @@ Parameters:
       "ghcr.io/openclarity/grype-server:v0.5.0" will be used if not overridden.
     Type: String
     Default: ''
+  YaraRuleServerContainerImageOverride:
+    Description: >
+      Name of the container image used for the yara rule server.
+      "ghcr.io/openclarity/yara-rule-server:v0.1.0" will be used if not overridden.
+    Type: String
+    Default: ''
   ExploitDBServerContainerImageOverride:
     Description: >
       Name of the container image used for the exploit db server.
@@ -1162,6 +1222,7 @@ Metadata:
           - ScannerContainerImageOverride
           - TrivyServerContainerImageOverride
           - GrypeServerContainerImageOverride
+          - YaraRuleServerContainerImageOverride
           - ExploitDBServerContainerImageOverride
           - PostgresqlContainerImageOverride
           - FreshclamMirrorContainerImageOverride
@@ -1185,6 +1246,8 @@ Metadata:
         default: Trivy Server Container Image Override
       GrypeServerContainerImageOverride:
         default: Grype Server Container Image Override
+      YaraRuleServerContainerImageOverride:
+        default: Yara Rule Server Container Image Override
       ExploitDBServerContainerImageOverride:
         default: Exploit DB Server Container Image Override
       FreshclamMirrorContainerImageOverride:
@@ -1307,6 +1370,10 @@ Conditions:
     - !Equals
       - !Ref GrypeServerContainerImageOverride
       - ''
+  YaraRuleServerContainerImageOverridden: !Not
+    - !Equals
+      - !Ref YaraRuleServerContainerImageOverride
+      - ''
   ExploitDBServerContainerImageOverridden: !Not
     - !Equals
       - !Ref ExploitDBServerContainerImageOverride

diff --git a/installation/azure/vmclarity-UI.json b/installation/azure/vmclarity-UI.json
@@ -263,6 +263,22 @@
                             "infoMessages": [],
                             "visible": true
                         },
+                        {
+                            "name": "yaraRuleServerContainerImage",
+                            "type": "Microsoft.Common.TextBox",
+                            "label": "Yara Rule Server Container Image",
+                            "subLabel": "",
+                            "defaultValue": "ghcr.io/openclarity/yara-rule-server:v0.1.0",
+                            "toolTip": "Yara Rule Server Container Image",
+                            "constraints": {
+                                "required": false,
+                                "regex": "",
+                                "validationMessage": "",
+                                "validations": []
+                            },
+                            "infoMessages": [],
+                            "visible": true
+                        },
                         {
                             "name": "assetScanDeletePolicy",
                             "type": "Microsoft.Common.DropDown",
@@ -470,6 +486,7 @@
                 "grypeServerContainerImage": "[steps('advanced').grypeServerContainerImage]",
                 "exploitDBContainerImage": "[steps('advanced').exploitDBContainerImage]",
                 "freshclamMirrorContainerImage": "[steps('advanced').freshclamMirrorContainerImage]",
+                "yaraRuleServerContainerImage": "[steps('advanced').yaraRuleServerContainerImage]",
                 "assetScanDeletePolicy": "[steps('advanced').assetScanDeletePolicy]",
                 "databaseToUse": "[steps('advanced').databaseSection.databaseToUse]",
                 "postgresContainerImage": "[steps('advanced').databaseSection.postgresContainerImage]",

diff --git a/installation/azure/vmclarity-install.sh b/installation/azure/vmclarity-install.sh
@@ -81,12 +81,28 @@ systemctl daemon-reload
 # Create directory for trivy server
 /usr/bin/mkdir -p /opt/trivy-server
 
+# Create directory for yara rule server
+/usr/bin/mkdir -p /opt/yara-rule-server
+
 # Enable and start/restart VMClarity backend
 systemctl enable vmclarity.service
 systemctl restart vmclarity.service
 EOF
 chmod 744 /etc/vmclarity/deploy.sh
 
+cat << 'EOF' > /etc/vmclarity/yara-rule-server.yaml
+enable_json_log: true
+rule_update_schedule: "0 0 * * *"
+rule_sources:
+  - name: "base"
+    url: "https://github.com/Yara-Rules/rules/archive/refs/heads/master.zip"
+    exclude_regex: ".*index.*.yar|.*/utils/.*|.*/deprecated/.*|.*index_.*|.*MALW_AZORULT.yar"
+  - name: "magic"
+    url: "https://github.com/securitymagic/yara/archive/refs/heads/main.zip"
+    exclude_regex: ".*index.*.yar"
+EOF
+chmod 644 /etc/vmclarity/yara-rule-server.yaml
+
 cat << 'EOF' > /etc/vmclarity/orchestrator.env
 PROVIDER=Azure
 VMCLARITY_AZURE_SUBSCRIPTION_ID=__AZURE_SUBSCRIPTION_ID__
@@ -261,6 +277,27 @@ services:
       restart_policy:
         condition: on-failure
 
+  yara-rule-server:
+    image: __YaraRuleServerContainerImage__
+    command:
+      - run
+    ports:
+      - "9993:8080"
+    configs:
+      - source: yara_rule_server_config
+        target: /etc/yara-rule-server/config.yaml
+    volumes:
+      - type: bind
+        source: /opt/yara-rule-server
+        target: /var/lib/yara-rule-server
+    logging:
+      driver: journald
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+
   postgresql:
     image: __PostgresqlContainerImage__
     env_file: ./postgres.env
@@ -289,6 +326,8 @@ configs:
     file: ./gateway.conf
   swagger_config:
     file: ./swagger-config.json
+  yara_rule_server_config:
+    file: ./yara-rule-server.yaml
 EOF
 
 cat << 'EOF' > /etc/vmclarity/swagger-config.json