From 6f8d71bdacdb3a068f9dd2d788496aa9c859f1eb Mon Sep 17 00:00:00 2001 From: Mike Terhar Date: Thu, 5 Dec 2024 06:04:03 +0000 Subject: [PATCH 1/4] Add the devcontainer setup used for otel pipelines --- devcontainer/README.md | 127 +++++++++++++++++++++++++++ devcontainer/collector/firstcol.yaml | 52 +++++++++++ devcontainer/devcontainer.json | 20 +++++ devcontainer/docker-compose-apps.yml | 35 ++++++++ devcontainer/docker-compose.yml | 12 +++ devcontainer/postCreateCommand.sh | 2 + devcontainer/refinery/config.yaml | 62 +++++++++++++ devcontainer/refinery/rules.yaml | 7 ++ 8 files changed, 317 insertions(+) create mode 100644 devcontainer/README.md create mode 100644 devcontainer/collector/firstcol.yaml create mode 100644 devcontainer/devcontainer.json create mode 100644 devcontainer/docker-compose-apps.yml create mode 100644 devcontainer/docker-compose.yml create mode 100644 devcontainer/postCreateCommand.sh create mode 100644 devcontainer/refinery/config.yaml create mode 100644 devcontainer/refinery/rules.yaml diff --git a/devcontainer/README.md b/devcontainer/README.md new file mode 100644 index 0000000..ea51265 --- /dev/null +++ b/devcontainer/README.md @@ -0,0 +1,127 @@ +# Devcontiner for Otel Pipelines + +This came from building custom collectors, tweak rules in collector configurations, and do all sorts of stuff to Refinery. + +This is improving on the `otel-playground` which was fine for yaml engineering but when it's Go time, this is way better. + +## What's in the box? + +Take this folder and put all of the contents into your workspace's .devcontainer folder. + +I've used this with VS Code on MacOS with Docker so I know it works under those conditions. + +### Containers + +1. firstcol is an otel collector image +2. refinery1 is a refinery image +3. workspace is a go development environment that can be used to build run refinery or collectors from source + +Since this is docker compose defined, you could add stuff from the otel demo or other apps or backends. + +### SSH/GitHub + +For the git commits to make sense, you need to run: + +```shell +git config --file ~/.gitconfig.local user.name "Mike Terhar" +git config --file ~/.gitconfig.local user.email "mike@terhar.com" +``` + +And if you want to get credit for the commits, you may want to change the values. + +I just throw my dotfiles at the new container by doing `curl -SsL https://mjt.sh | sh` + +If you've run `ssh-add` in your local machine terminal, it'll have GitHub access via the SSH agent. + +## Dev Container capabilities + +From the dev container json, you can see that it has a docker and vim plugin and runs a `postCreateCommand.sh`. +The `postCreateCommand.sh` installs Delve and the Otel Collector Builder. +The image it's based on is specified in the `docker-compose.yml` file `mcr.microsoft.com/devcontainers/go:1.23-bookworm`. +It seems to have some good stuff in it but I honestly have no idea if it's a good image beyond being able to do what I was doing. + +I mainly run: + +* Go [build, mod, run, etc] +* curl +* vim for some reason even though there's like a whole text editor thing going on +* docker + +All of those commands work great. + +## Volumes and configs + +There are 2 directories in here, one for the `firstcol` collector config and one for the `refinery1` config and rules files. +Feel free to change the inputs and outputs and sampling and transforms. + +Most of the config changes I did while working on things were automatically picked up. +I was recompiling the collector I was building but any changes to the inbound and outbound sides were picked up without restarting. + +If you have to restart, you can just use `docker stop` and `docker start` to get them to reload parts of the config that aren't hot-loadable. + +### Changing the devcontainer or docker compose configurations + +If you want to change an image to a newer one, edit the docker compose files and a little box will pop up in the corner to ask you to rebuild. +If it doesn't pop up, go to "close window" and after about 20 seconds of refinery delaying dying, all the containers will stop. +When you bring it back up it should ask about rebuilding. + +If you want to connect VS Code to a different container but keep the proper dev container running, just add a new one and set `service` to the new dev container's name. + +## How to run end-to-end tests? + +Loadgen is good. + +### Replaying Otel from S3 with Curl + +From inside your IDE's devcontainer terminal, you can run curl and reference the first collector by docker's internal DNS. + +```shell +curl -X POST http://firstcol:4318/v1/traces -d @otlp_trace.json -H "CONTENT-TYPE: application/json" -H "x-honeycomb-team: hcaik_xxxxxxxxxxxxxxxxxxxxxx" +``` + +## Sending spans does what? + +Firstcol gets it on port 4318. Note that's the internal port rather than the one docker compose exposes to the host. + +It then is processed there and logs fall out into docker's std-out. To see them, you can run `docker ps` and get the container IDs. + +```shell +docker logs otel-collector-dev_devcontainer-firstcol-1 -f +``` + +You can do the same to refinery, but I usually send those logs off to Honeycomb. + +```shell +docker logs otel-collector-dev_devcontainer-refinery1-1 -f +``` + +And because those images are scratch images, you can't shell into them or do anything helpful. +If they get bad, you're just going to have to start the whole dev environment up again. +Which should be painless!!! + +If you want to see what is happening inside the app you're using, Delve can get you some cool insights and break points. + +Make a `.vscode/launch.json` file and put this in it. + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Connect to server", + "type": "go", + "request": "attach", + "mode": "remote", + "port": 2345, + "host": "127.0.0.1", + "apiVersion": 2, + "showLog": true + } + ] +} +``` + +## Downsides + +1. If you run an awesome command and want to use it later, save it into a readme or something. History is fleeting. +2. When you do a big rebuild, it has to re-download all the go modules and such. An additional mount could probably solve this but it may make my host machine messier and that's the opposite of the goal here. \ No newline at end of file diff --git a/devcontainer/collector/firstcol.yaml b/devcontainer/collector/firstcol.yaml new file mode 100644 index 0000000..bcecaf2 --- /dev/null +++ b/devcontainer/collector/firstcol.yaml @@ -0,0 +1,52 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + include_metadata: true + +processors: + batch: + metadata_keys: + - x-honeycomb-team + metadata_cardinality_limit: 30 + + transform: + error_mode: ignore + trace_statements: + - context: resource + statements: + - set(attributes["collector.before_refinery"], "${env:HOSTNAME}") + +exporters: + debug: + verbosity: detailed + file: + path: ./output.json + otlphttp/refinery: + endpoint: "http://refinery1:8080" + auth: + authenticator: headers_setter + +extensions: + headers_setter: + headers: + - action: upsert + key: x-honeycomb-team + from_context: x-honeycomb-team + +service: + extensions: + - headers_setter + pipelines: + traces: + receivers: [otlp] + processors: [transform, batch] + exporters: [otlphttp/refinery] + logs: + receivers: [otlp] + processors: [transform, batch] + exporters: [otlphttp/refinery] + telemetry: + logs: + level: debug \ No newline at end of file diff --git a/devcontainer/devcontainer.json b/devcontainer/devcontainer.json new file mode 100644 index 0000000..a72a676 --- /dev/null +++ b/devcontainer/devcontainer.json @@ -0,0 +1,20 @@ +{ + "name": "Go", + "dockerComposeFile": [ + "./docker-compose-apps.yml", + "./docker-compose.yml" + ], + "service": "workspace", + "forwardPorts": [ + 4317, + 4318, + 8088 + ], + "postCreateCommand": "bash .devcontainer/postCreateCommand.sh", + "shutdownAction": "stopCompose", + "workspaceFolder": "/workspace", + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}, + "ghcr.io/guiyomh/features/vim:0": {} + } +} \ No newline at end of file diff --git a/devcontainer/docker-compose-apps.yml b/devcontainer/docker-compose-apps.yml new file mode 100644 index 0000000..dfe972a --- /dev/null +++ b/devcontainer/docker-compose-apps.yml @@ -0,0 +1,35 @@ +services: + firstcol: + image: otel/opentelemetry-collector-k8s:0.113.0 + environment: + - HOSTNAME=firstcol + ports: + - 14317:4317 + - 14318:4318 + volumes: + - ./collector:/collector + entrypoint: + - "/otelcol-k8s" + command: + - "--config=/collector/firstcol.yaml" + refinery1: + image: honeycombio/refinery:2.9.0 + environment: + - OTEL_RESOURCE_ATTRIBUTES=cluster.name=devcontainer + - REFINERY_QUERY_AUTH_TOKEN=xxxxxxxxxxxxxxxxxxx + - REFINERY_HONEYCOMB_API_KEY=hcaik_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + ports: + - 8889:8080 + entrypoint: + - "refinery" + command: + - "-c" + - "/etc/refinery/config.yaml" + - "-r" + - "/etc/refinery/rules.yaml" + volumes: + - ./refinery:/etc/refinery + redis: + image: redis:7.2 + expose: + - "6379" diff --git a/devcontainer/docker-compose.yml b/devcontainer/docker-compose.yml new file mode 100644 index 0000000..2ef6f18 --- /dev/null +++ b/devcontainer/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3' +services: + workspace: + image: "mcr.microsoft.com/devcontainers/go:1.23-bookworm" + volumes: + - ..:/workspace:cached + - /var/run/docker.sock:/var/run/docker.sock + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined + command: /bin/sh -c "while sleep 1000; do :; done" \ No newline at end of file diff --git a/devcontainer/postCreateCommand.sh b/devcontainer/postCreateCommand.sh new file mode 100644 index 0000000..53398cf --- /dev/null +++ b/devcontainer/postCreateCommand.sh @@ -0,0 +1,2 @@ +go install go.opentelemetry.io/collector/cmd/builder@latest +go install github.com/go-delve/delve/cmd/dlv@latest \ No newline at end of file diff --git a/devcontainer/refinery/config.yaml b/devcontainer/refinery/config.yaml new file mode 100644 index 0000000..5e0db96 --- /dev/null +++ b/devcontainer/refinery/config.yaml @@ -0,0 +1,62 @@ +Network: + ListenAddr: 0.0.0.0:8080 + PeerListenAddr: 0.0.0.0:8081 + HoneycombAPI: http://workspace:8088 +GRPCServerParameters: + Enabled: true + ListenAddr: 0.0.0.0:4317 + MaxRecvMsgSize: 15MB +AccessKeys: + AcceptOnlyListedKeys: false +Collection: + AvailableMemory: 4GB + MaxMemoryPercentage: 88 + CacheCapacity: 200_000 + IncomingQueueSize: 500_000 + PeerQueueSize: 500_000 +BufferSizes: + UpstreamBufferSize: 500_000 + PeerBufferSize: 500_000 +Debugging: + AdditionalErrorFields: + - collector.name + - service.name + - trace.trace_id + - trace.span_id + - trace.parent_id + - name +General: + ConfigurationVersion: 2 + MinRefineryVersion: v2.0 +Logger: + Type: honeycomb + Level: info +HoneycombLogger: + Dataset: refinery-logs + SamplerEnabled: false +LegacyMetrics: + Dataset: refinery-metrics + Enabled: true + ReportingInterval: 30s +OTelMetrics: + Dataset: refinery-otel-metrics + Enabled: true + ReportingInterval: 30s +PrometheusMetrics: + Enabled: true +RefineryTelemetry: + AddRuleReasonToTrace: true + AddCountsToRoot: true +StressRelief: + ActivationLevel: 95 + DeactivationLevel: 70 + Mode: never + MinimumActivationDuration: 4s +Traces: + SendDelay: 3s + TraceTimeout: 5s +OTelTracing: + Enabled: true + APIKey: hcaik_xxxxxxxxxxxxxxxxxxxxxxxxxx + SampleRate: 1 + Dataset: "refinery-traces" \ No newline at end of file diff --git a/devcontainer/refinery/rules.yaml b/devcontainer/refinery/rules.yaml new file mode 100644 index 0000000..360dff9 --- /dev/null +++ b/devcontainer/refinery/rules.yaml @@ -0,0 +1,7 @@ +RulesVersion: 2 +Samplers: + __default__: + RulesBasedSampler: + Rules: + - Name: Everything + SampleRate: 1 \ No newline at end of file From 218e693e2dbff8a1b4236828b0072d125322db64 Mon Sep 17 00:00:00 2001 From: Mike Terhar Date: Thu, 5 Dec 2024 15:00:47 -0500 Subject: [PATCH 2/4] Comment out OTelTracing section Allows the defaults to start up without the container dying. --- devcontainer/refinery/config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/devcontainer/refinery/config.yaml b/devcontainer/refinery/config.yaml index 5e0db96..11cfd15 100644 --- a/devcontainer/refinery/config.yaml +++ b/devcontainer/refinery/config.yaml @@ -55,8 +55,8 @@ StressRelief: Traces: SendDelay: 3s TraceTimeout: 5s -OTelTracing: - Enabled: true - APIKey: hcaik_xxxxxxxxxxxxxxxxxxxxxxxxxx - SampleRate: 1 - Dataset: "refinery-traces" \ No newline at end of file +#OTelTracing: +# Enabled: true +# APIKey: hcaik_xxxxxxxxxxxxxxxxxxxxxxxxxx +# SampleRate: 1 +# Dataset: "refinery-traces" From 2bd2b180d4cd450b541dd2d09295e767d1dd5604 Mon Sep 17 00:00:00 2001 From: Mike Terhar Date: Tue, 10 Dec 2024 18:46:01 -0500 Subject: [PATCH 3/4] set workspace contsiner version to have golang 1.22.10 --- devcontainer/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devcontainer/docker-compose.yml b/devcontainer/docker-compose.yml index 2ef6f18..8c993a5 100644 --- a/devcontainer/docker-compose.yml +++ b/devcontainer/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: workspace: - image: "mcr.microsoft.com/devcontainers/go:1.23-bookworm" + image: "mcr.microsoft.com/devcontainers/go:dev-1.22-bookworm" volumes: - ..:/workspace:cached - /var/run/docker.sock:/var/run/docker.sock From 2a3d790503295e73ef804cb3fd60347221ac2925 Mon Sep 17 00:00:00 2001 From: Mike Terhar Date: Tue, 10 Dec 2024 18:51:01 -0500 Subject: [PATCH 4/4] Add .devcontainer to avoid misunderstandings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The devconainer directory isn’t for this repo, it’s just an example. If someone copies it and names it .devcontainer for this repo we don’t want it showing up in their commits. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 29ef5c2..01a585b 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ stack_parameters.json # Python **/venv/** + +.devcontainer