[CORE] KubeHound graph DSL (#94)

.github/workflows/docker.yml

@@ -1,10 +1,10 @@
-name: docker
+name: build-kubegraph-image
 
 on:
   workflow_dispatch:
-  # push:
-  #   tags:
-  #     - "*"
+  push:
+    tags:
+      - "v*"
 
 env:
   REGISTRY: ghcr.io
@@ -20,13 +20,13 @@ jobs:
         # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
         matrix:
           include:
-            - dockerfile: ./Dockerfile
-              component: core
-            - dockerfile: deployments/kubehound/janusgraph/Dockerfile
-              image: janusgraph
+            - dockerfile: deployments/kubehound/kubegraph/Dockerfile
+              image: graph
+              workdir: deployments/kubehound/kubegraph/
     permissions:
       contents: read
       packages: write
+
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09
@@ -56,14 +56,33 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2.9.1
+
       - name: Build and push Docker image
+        if: ${{ github.event_name == 'push' }}
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
-          context: .
+          context: ${{ matrix.workdir }}
+          platforms: linux/amd64,linux/arm64
           file: ${{ matrix.dockerfile }}
           push: true
           build-args: |
             VERSION=${{ github.ref_name }}
           tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image }}:${{ github.ref_name }}
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image }}:latest
+              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image }}:${{ github.ref_name }}
+              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image }}:latest
+
+      - name: Build and push Docker image
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: ${{ matrix.workdir }}
+          platforms: linux/amd64,linux/arm64
+          file: ${{ matrix.dockerfile }}
+          push: true
+          build-args: |
+            VERSION=${{ github.sha }}
+          tags: |
+              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image }}:snapshot-${{ github.sha }}
+              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image }}:latest

.github/workflows/docs.yml

@@ -30,5 +30,5 @@ jobs:
       - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1
         with:
           python-version: 3.x
-      - run: pip install mkdocs-material mkdocs-awesome-pages-plugin
+      - run: pip install mkdocs-material mkdocs-awesome-pages-plugin markdown-captions
       - run: mkdocs gh-deploy --force

.github/workflows/linter.yml

@@ -1,4 +1,4 @@
-name: KubeHound Linter
+name: kubehound-linter
 
 on:
   push:

.github/workflows/release.yml

@@ -1,9 +1,9 @@
-name: KubeHound Release
+name: kubehound-release
 
 on:
   push:
     tags:
-      - "*"
+      - "v*"
 
 permissions:
   contents: read

.github/workflows/system-test.yml

@@ -1,4 +1,4 @@
-name: KubeHound System Tests
+name: kubehound-system-tests
 
 on:
   push:

.github/workflows/unit-test.yml

@@ -1,4 +1,4 @@
-name: KubeHound Unit Tests
+name: kubehound-unit-tests
 
 on:
   push:

.gitignore

@@ -8,6 +8,9 @@ bin/
 *.dll
 *.so
 *.dylib
+*.jar
+*.class
+*.lst
 
 # Test binary, built with `go test -c`
 *.test
@@ -41,4 +44,7 @@ test/setup/.kube
 test/system/generator/generator
 *.env
 *.kube-config
-scripts/collectors/
+scripts/collectors/
+
+# java
+deployments/kubehound/kubegraph/dsl/kubehound/target

Makefile

@@ -36,8 +36,8 @@ ifndef KUBEHOUND_ENV
 endif
 
 ifeq (,$(filter $(SYSTEM_TEST_CMD),$(MAKECMDGOALS)))
-	ifeq (${KUBEHOUND_ENV}, prod)
-		DOCKER_COMPOSE_FILE_PATH += -f deployments/kubehound/docker-compose.prod.yaml
+	ifeq (${KUBEHOUND_ENV}, release)
+		DOCKER_COMPOSE_FILE_PATH += -f deployments/kubehound/docker-compose.release.yaml
 	else ifeq (${KUBEHOUND_ENV}, dev)
 		DOCKER_COMPOSE_FILE_PATH += -f deployments/kubehound/docker-compose.dev.yaml
 	endif
@@ -107,7 +107,7 @@ ifndef KUBEHOUND_ENV
 	$(error KUBEHOUND_ENV is undefined)
 endif
 	$(DOCKER_CMD) volume rm kubehound-${KUBEHOUND_ENV}_mongodb_data
-	$(DOCKER_CMD) volume rm kubehound-${KUBEHOUND_ENV}_janusgraph_data
+	$(DOCKER_CMD) volume rm kubehound-${KUBEHOUND_ENV}_kubegraph_data
 
 .PHONY: backend-reset-hard
 backend-reset-hard: | backend-down backend-wipe backend-up ## Restart the kubehound stack and wipe all data
@@ -151,7 +151,16 @@ help: ## Show this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(HELP_MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: thirdparty-licenses
-thirdparty-licenses:
+thirdparty-licenses: ## Generate the list of 3rd party dependencies and write to LICENSE-3rdparty.csv
 	go get github.com/google/go-licenses
 	go install github.com/google/go-licenses
 	$(GOPATH)/bin/go-licenses csv github.com/DataDog/KubeHound/cmd/kubehound | sort > $(ROOT_DIR)/LICENSE-3rdparty.csv
+
+.PHONY: local-wiki
+local-wiki: ## Generate and serve the mkdocs wiki on localhost
+	pip install mkdocs-material mkdocs-awesome-pages-plugin markdown-captions
+	mkdocs serve
+
+.PHONY: local-release
+local-release: ## Generate release packages locally via goreleaser
+	goreleaser release --snapshot --clean --config .goreleaser.yaml

README.md

@@ -1,7 +1,28 @@
-# KubeHound
+# KubeHound  <!-- omit in toc -->
 
 A Kubernetes attack graph tool allowing automated calculation of attack paths between assets in a cluster
 
+- [Quick Links](#quick-links)
+- [Requirements](#requirements)
+  - [Application](#application)
+  - [Test (Development only)](#test-development-only)
+- [Quick Start](#quick-start)
+  - [Prebuilt Releases](#prebuilt-releases)
+  - [From Source](#from-source)
+  - [Sample Data](#sample-data)
+- [Advanced Usage](#advanced-usage)
+  - [Infrastructure Setup](#infrastructure-setup)
+  - [Running Kubehound](#running-kubehound)
+- [Using KubeHound Data](#using-kubehound-data)
+- [Development](#development)
+  - [Build](#build)
+  - [Release build](#release-build)
+  - [Unit Testing](#unit-testing)
+  - [System Testing](#system-testing)
+    - [Environment variable:](#environment-variable)
+    - [Setup](#setup)
+    - [CI Testing](#ci-testing)
+
 ![Example Path](./docs/images/example-graph.png)
 
 ## Quick Links
@@ -33,6 +54,8 @@ Release binaries are available for Linux / Windows / Mac OS via the [releases](h
 ./kubehound.sh backend-up
 ```
 
+*NOTE*: you must have [setup GHCR access](https://codefresh.io/docs/docs/integrations/docker-registries/github-container-registry/)
+
 Next choose a target Kubernetes cluster, either:
 
 * Select the targeted cluster via `kubectx` (need to be installed separately)     
@@ -97,13 +120,12 @@ cp deployments/kubehound/.env.tpl deployments/kubehound/.env
 
 Edit the variables (datadog env `DD_*` related and `KUBEHOUND_ENV`):
 
-* `KUBEHOUND_ENV`: `dev` or `prod` 
+* `KUBEHOUND_ENV`: `dev` or `release` 
 * `DD_API_KEY`: api key you created from https://app.datadoghq.com/ website
 
 Note:
-* `KUBEHOUND_ENV=prod` will use prebuilt image from ghcr.io (:rotating_light: currently NOT supported :rotating_light:)
-* `KUBEHOUND_ENV=dev` will build the images locally
-
+* `KUBEHOUND_ENV=dev` will build the images locally (and provide some local debugging containers e.g `mongo-express`)
+* `KUBEHOUND_ENV=release` will use prebuilt images from ghcr.io 
 
 ### Running Kubehound
 
@@ -163,6 +185,14 @@ make build
 
 All binaries will be output to the [bin](./bin/) folder
 
+### Release build
+
+Build the release packages locally using [goreleaser](https://goreleaser.com/install):
+
+```bash
+make local-release
+```
+
 ### Unit Testing
 
 The full suite of unit tests can be run locally via:

configs/etc/kubehound-reference.yaml

@@ -75,6 +75,9 @@ builder:
   # vertex:
   #   # Batch size for vertex inserts
   #   batch_size: 500
+  # 
+  #   # Small batch size for vertex inserts
+  #   batch_size_small: 100
 
   # Edge builder configuration
   edge:

deployments/kubehound/docker-compose.dev.yaml

@@ -7,12 +7,13 @@ services:
     volumes:
       - mongodb_data:/data/db
 
-  janusgraph:
+  kubegraph:
+    build: ./kubegraph/
     ports:
       - "127.0.0.1:8182:8182"
       - "127.0.0.1:8099:8099"
     volumes:
-      - janusgraph_data:/var/lib/janusgraph
+      - kubegraph_data:/var/lib/janusgraph
 
   mongo-express:
     image: mongo-express:1.0.0-alpha
@@ -27,26 +28,10 @@ services:
       - kubenet
     environment:
       - ME_CONFIG_MONGODB_SERVER=mongodb
-
-  kubehound:
-    container_name: ${COMPOSE_PROJECT_NAME}-core
-    profiles: ["core"]
-    build:
-      context: ../../
-      dockerfile: Dockerfile
-    environment:
-      - KUBECONFIG=/tmp/.kube/config
-    volumes:
-      - ./.kube-config:/tmp/.kube/config
-    networks:
-      - kubenet
-      - kind
-    labels:
-      com.datadoghq.ad.logs: '[{"app": "core", "service": "kubehound"}]'
 
 volumes:
   mongodb_data:
-  janusgraph_data:
+  kubegraph_data:
 
 networks:
   kind:

deployments/kubehound/docker-compose.release.yaml

@@ -7,16 +7,17 @@ services:
     volumes:
       - mongodb_data:/data/db
 
-  janusgraph:
+  kubegraph:
+    image: ghcr.io/datadog/kubehound-graph:latest
     ports:
       - "127.0.0.1:8182:8182"
       - "127.0.0.1:8099:8099"
     volumes:
-      - janusgraph_data:/var/lib/janusgraph
+      - kubegraph_data:/var/lib/janusgraph
 
 volumes:
   mongodb_data:
-  janusgraph_data:
+  kubegraph_data:
 
 networks:
   kind:

deployments/kubehound/docker-compose.testing.yaml

@@ -4,7 +4,8 @@ services:
   mongodb:
     ports:
       - "127.0.0.1:27018:27017"
-  janusgraph:
+  kubegraph:
+    build: ./kubegraph/
     ports:
       - "127.0.0.1:8183:8182"
 networks: