From 6f6dbc1aa92c3bb89ca9d615193f1afeaee542be Mon Sep 17 00:00:00 2001 From: Philip Schmid Date: Tue, 14 Nov 2023 14:55:29 +0100 Subject: [PATCH] Added netpol examples without host policies Signed-off-by: Philip Schmid --- .github/dependabot.yml | 13 ++ .github/workflows/github-actions.yml | 53 ++++++ README.md | 5 +- deploy/.terraform-docs.yaml | 7 + deploy/01-vpc.tf | 2 +- deploy/02-kubeadm.tf | 2 +- deploy/03-cilium-values-1.14.yaml | 32 ++-- deploy/03-cilium.tf | 3 +- deploy/README.md | 10 +- deploy/manifests/goldpinger/goldpinger.yaml | 2 +- netpols/no-host-policies/README.md | 106 ++++++++++++ .../ccnp-global-infra.yaml | 4 +- .../cnp-infra-cert-manager.yaml | 4 +- .../cnp-infra-goldpinger.yaml | 38 +++++ .../cnp-infra-ingress-nginx.yaml | 2 +- .../cnp-infra-kube-system.yaml | 2 +- .../cnp-infra-monitoring-stack.yaml | 11 +- .../cnp-user-template.yaml | 2 +- .../demo-app-podinfo.sh | 0 netpols/{ => with-host-policies}/README.md | 2 +- .../with-host-policies/ccnp-global-infra.yaml | 77 +++++++++ .../ccnp-host-all.yaml | 2 +- .../ccnp-host-cp.yaml | 14 +- .../ccnp-host-infra.yaml | 2 +- .../cnp-infra-cert-manager.yaml | 92 +++++++++++ .../cnp-infra-goldpinger.yaml | 2 +- .../cnp-infra-ingress-nginx.yaml | 71 ++++++++ .../cnp-infra-kube-system.yaml | 36 ++++ .../cnp-infra-monitoring-stack.yaml | 156 ++++++++++++++++++ .../with-host-policies/cnp-user-template.yaml | 35 ++++ .../with-host-policies/demo-app-podinfo.sh | 56 +++++++ .../host-enforcement-status.sh | 0 .../host-enforcement-to-audit.sh | 0 .../host-enforcement-to-enforce.sh | 0 .../host-enforcement-verification.sh | 0 35 files changed, 798 insertions(+), 45 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/github-actions.yml create mode 100644 deploy/.terraform-docs.yaml create mode 100644 netpols/no-host-policies/README.md rename netpols/{ => no-host-policies}/ccnp-global-infra.yaml (86%) rename netpols/{ => no-host-policies}/cnp-infra-cert-manager.yaml (92%) create mode 100644 netpols/no-host-policies/cnp-infra-goldpinger.yaml rename netpols/{ => no-host-policies}/cnp-infra-ingress-nginx.yaml (98%) rename netpols/{ => no-host-policies}/cnp-infra-kube-system.yaml (97%) rename netpols/{ => no-host-policies}/cnp-infra-monitoring-stack.yaml (94%) rename netpols/{ => no-host-policies}/cnp-user-template.yaml (96%) rename netpols/{ => no-host-policies}/demo-app-podinfo.sh (100%) rename netpols/{ => with-host-policies}/README.md (99%) create mode 100644 netpols/with-host-policies/ccnp-global-infra.yaml rename netpols/{ => with-host-policies}/ccnp-host-all.yaml (98%) rename netpols/{ => with-host-policies}/ccnp-host-cp.yaml (77%) rename netpols/{ => with-host-policies}/ccnp-host-infra.yaml (95%) create mode 100644 netpols/with-host-policies/cnp-infra-cert-manager.yaml rename netpols/{ => with-host-policies}/cnp-infra-goldpinger.yaml (96%) create mode 100644 netpols/with-host-policies/cnp-infra-ingress-nginx.yaml create mode 100644 netpols/with-host-policies/cnp-infra-kube-system.yaml create mode 100644 netpols/with-host-policies/cnp-infra-monitoring-stack.yaml create mode 100644 netpols/with-host-policies/cnp-user-template.yaml create mode 100755 netpols/with-host-policies/demo-app-podinfo.sh rename netpols/{ => with-host-policies}/host-enforcement-status.sh (100%) rename netpols/{ => with-host-policies}/host-enforcement-to-audit.sh (100%) rename netpols/{ => with-host-policies}/host-enforcement-to-enforce.sh (100%) rename netpols/{ => with-host-policies}/host-enforcement-verification.sh (100%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..af3db4e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: "terraform" + directory: /deploy + schedule: + interval: "daily" + time: "23:00" + timezone: "Europe/Zurich" + open-pull-requests-limit: 3 + rebase-strategy: "disabled" + labels: + - ci/dependabot + - kind/enhancement \ No newline at end of file diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml new file mode 100644 index 0000000..1794f7d --- /dev/null +++ b/.github/workflows/github-actions.yml @@ -0,0 +1,53 @@ +name: Validation Actions +on: + pull_request_target: + types: + - opened + - synchronize + - reopened + push: + branches: + - main +jobs: + formatting: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ github.sha }} + - name: terraform fmt + uses: dflook/terraform-fmt-check@529e30563b2c558dc0b8c450b5cec1cc93bd7fe4 + with: + path: /deploy + docs: + runs-on: ubuntu-22.04 + needs: formatting + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ github.ref }} + - name: Render terraform docs inside the README.md and push changes back to PR branch + uses: terraform-docs/gh-actions@d1c99433f7a1e5003ef213d70f89aaa47cb0b675 + with: + working-dir: /deploy + output-file: README.md + output-method: inject + output-format: markdown table + indention: 3 + git-push: "true" + validate-netpol-yamls: + runs-on: ubuntu-22.04 + needs: formatting + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - name: Validate Network Policy YAML files + run: yamllint netpols/*/*.yaml + validate-cilium-valuesyaml: + runs-on: ubuntu-22.04 + needs: formatting + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - name: Validate Cilium Helm values YAML file + run: yamllint deploy/03-cilium-values-1.14.yaml diff --git a/README.md b/README.md index 9c9eb12..b36be66 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,11 @@ This is a demo of how to lock down your Kubernetes cluster using advanced [Ciliu ![architecture overview](pictures/architecture-overview.jpg) -If you would like to apply those or similar policies to your existing clusters, it's still possible without too much effort by leveraging Cilium Hubble's visibility capabilities to see if a newly introduced Cilium Network Policy causes unwanted traffic denies or not. You should check out our free [Isovalent Zero Trust hands-on lab](https://isovalent.com/labs/cilium-enterprise-zero-trust-visibility/) in case you are eager to learn more about our recommended way of validating new Cilium Network Policies before applying them to existing clusters. +If you would like to apply those or similar policies to your existing clusters, it's still possible without too much effort by leveraging Cilium Hubble's visibility capabilities to see if a newly introduced Cilium Network Policy causes unwanted traffic denies or not. You should check out Isovalent's free [Zero Trust hands-on lab](https://isovalent.com/labs/cilium-enterprise-zero-trust-visibility/) in case you are eager to learn more about our recommended way of validating new Cilium Network Policies before applying them to existing clusters. * Go to the `slides` directory to see slide decks of talks I did based on this demo setup. * Head over to the `deploy` directory to see how the demo Kubeadm Kubernetes cluster and infrastructure components are deployed. -* Check the `netpols` directory to see the actual Cilium (Cluster-wide) Network Policies. +* Check the `netpols/no-host-policies` directory to see the actual Cilium (Cluster-wide) Network Policies. +* Check the `netpols/with-host-policies` directory to see the actual Cilium (Cluster-wide) Network Policies where Cilium Host Policies are used as well (Host Firewall). More examples and even hands-on labs on how to leverage Cilium Network Policies can be found in the free [Isovalent "Security Professional" learning track](https://isovalent.com/learning-tracks/#securityProfessionals). \ No newline at end of file diff --git a/deploy/.terraform-docs.yaml b/deploy/.terraform-docs.yaml new file mode 100644 index 0000000..c2555e7 --- /dev/null +++ b/deploy/.terraform-docs.yaml @@ -0,0 +1,7 @@ +# https://terraform-docs.io/user-guide/configuration/ +formatter: "markdown table" +output: + file: README.md + mode: inject +settings: + indent: 3 \ No newline at end of file diff --git a/deploy/01-vpc.tf b/deploy/01-vpc.tf index 86ce8b2..16bab51 100644 --- a/deploy/01-vpc.tf +++ b/deploy/01-vpc.tf @@ -5,7 +5,7 @@ resource "random_id" "cluster" { } module "vpc" { - source = "git::ssh://git@github.com/isovalent/terraform-aws-vpc.git?ref=v1.5" + source = "git::ssh://git@github.com/isovalent/terraform-aws-vpc.git?ref=v1.7" cidr = var.vpc_cidr name = "${var.cluster_name}-${random_id.cluster.dec}" diff --git a/deploy/02-kubeadm.tf b/deploy/02-kubeadm.tf index 253bb9c..a18b1cf 100644 --- a/deploy/02-kubeadm.tf +++ b/deploy/02-kubeadm.tf @@ -1,5 +1,5 @@ module "kubeadm" { - source = "git::ssh://git@github.com/isovalent/terraform-aws-kubeadm.git?ref=v2.4" + source = "git::ssh://git@github.com/isovalent/terraform-aws-kubeadm.git?ref=v2.6.1" vpc_id = module.vpc.id ami_name_filter = var.ami_name_filter diff --git a/deploy/03-cilium-values-1.14.yaml b/deploy/03-cilium-values-1.14.yaml index ea03276..85dd3e0 100644 --- a/deploy/03-cilium-values-1.14.yaml +++ b/deploy/03-cilium-values-1.14.yaml @@ -27,13 +27,13 @@ ipam: clusterPoolIPv4MaskSize: 25 # Routing/encapsulation mode -tunnel: vxlan +tunnelProtocol: "vxlan" +routingMode: "tunnel" # KubeProxyReplacement -kubeProxyReplacement: "strict" +kubeProxyReplacement: "true" k8sServiceHost: ${KUBE_APISERVER_HOST} k8sServicePort: ${KUBE_APISERVER_PORT} -kubeProxyReplacementHealthzBindAddr: "0.0.0.0:10256" # Enable support for Gateway API gatewayAPI: @@ -55,15 +55,17 @@ hubble: enabled: true metrics: enableOpenMetrics: true - # https://docs.cilium.io/en/stable/observability/metrics/#hubble-exported-metrics enabled: - - dns - - drop - - tcp - - flow - - port-distribution - - icmp - - httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction + # https://docs.cilium.io/en/stable/observability/metrics/#hubble-exported-metrics + # Remove `;query` from the `dns` line for production -> bad metrics cardinality + - dns:labelsContext=source_namespace,destination_namespace;query + - drop:labelsContext=source_namespace,destination_namespace + - tcp:labelsContext=source_namespace,destination_namespace + - port-distribution:labelsContext=source_namespace,destination_namespace + - icmp:labelsContext=source_namespace,destination_namespace;sourceContext=workload-name|reserved-identity;destinationContext=workload-name|reserved-identity + - flow:sourceContext=workload-name|reserved-identity;destinationContext=workload-name|reserved-identity;labelsContext=source_namespace,destination_namespace + - "httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction;sourceContext=workload-name|reserved-identity;destinationContext=workload-name|reserved-identity" + - "policy:sourceContext=app|workload-name|pod|reserved-identity;destinationContext=app|workload-name|pod|dns|reserved-identity;labelsContext=source_namespace,destination_namespace" serviceMonitor: enabled: true dashboards: @@ -147,7 +149,7 @@ operator: enabled: true serviceMonitor: enabled: true -# Operator Dashboards + # Operator Dashboards dashboards: enabled: true annotations: @@ -161,6 +163,6 @@ prometheus: # Cilium Agent Dashboards dashboards: - enabled: true - annotations: - grafana_folder: "Cilium Agent Dashboards" \ No newline at end of file + enabled: true + annotations: + grafana_folder: "Cilium Agent Dashboards" diff --git a/deploy/03-cilium.tf b/deploy/03-cilium.tf index fb24de3..580609b 100644 --- a/deploy/03-cilium.tf +++ b/deploy/03-cilium.tf @@ -1,11 +1,12 @@ module "cilium" { - source = "git::ssh://git@github.com/isovalent/terraform-k8s-cilium.git?ref=v1.3" + source = "git::ssh://git@github.com/isovalent/terraform-k8s-cilium.git?ref=v1.6.2" depends_on = [ module.kubeadm ] cilium_helm_release_name = "cilium" + wait_for_total_control_plane_nodes = true cilium_helm_values_file_path = var.cilium_helm_values_file_path cilium_helm_version = var.cilium_helm_version cilium_helm_chart = var.cilium_helm_chart diff --git a/deploy/README.md b/deploy/README.md index e6744a8..32cf5bb 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,4 +1,5 @@ # Demo Infrastructure Deployment + For this demo infrastructure, some Isovalent internal Terraform modules were used to spin up a Kubeadm-based K8s cluster and installing Cilium. Nevertheless, you can still basically do the same in a manual way by following the [kubeadm Cluster Setup](https://gist.github.com/PhilipSchmid/e34a725d5836d21432fd10b0709a5c4a) guide. Ensure you're aware of the following things: - kube-proxy shouldn't be installed, as we use Cilium's KubeProxyReplacement. @@ -9,6 +10,8 @@ Also, check the following files to see how things are deployed: - Cilium Helm values: `deploy/03-cilium-values-1.14.yaml` - Infrastructure components: `deploy/scripts/deploy-uc.sh` +## Troubleshooting + SSH access via SSH jumphost: ```bash ssh -i ~/.ssh/id_ed25519.pub \ @@ -20,4 +23,9 @@ ssh -i ~/.ssh/id_ed25519.pub \ -o UserKnownHostsFile=/dev/null \ rocky@" \ rocky@ -``` \ No newline at end of file +``` + +## Terraform Module Doc + + + \ No newline at end of file diff --git a/deploy/manifests/goldpinger/goldpinger.yaml b/deploy/manifests/goldpinger/goldpinger.yaml index 863badb..e0b45c8 100644 --- a/deploy/manifests/goldpinger/goldpinger.yaml +++ b/deploy/manifests/goldpinger/goldpinger.yaml @@ -76,7 +76,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: "docker.io/bloomberg/goldpinger:v3.7.0" + image: "docker.io/bloomberg/goldpinger:3.9.0" imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false diff --git a/netpols/no-host-policies/README.md b/netpols/no-host-policies/README.md new file mode 100644 index 0000000..35cd475 --- /dev/null +++ b/netpols/no-host-policies/README.md @@ -0,0 +1,106 @@ +# Demo Network Policies (without Host Policies) + +## Generic Hubble Configuration for Visibility +```bash +# - Since we enabled TLS for Hubble, we need to configure Hubble CLI accordingly +# - Get Hubble CLI from here: https://docs.cilium.io/en/stable/gettingstarted/hubble_setup/#install-the-hubble-client +hubble config set tls true +hubble config set tls-ca-cert-files /path/to/cilium-netpol-demo/deploy/cilium-ca-crt.pem +hubble config set tls-server-name "*.hubble-relay.cilium.io" +# Open the port-forwarding in a separate shell: +kubectl port-forward -n kube-system svc/hubble-relay 4245:443 +# Finally check Hubble CLI's connection and the flows: +hubble status +``` + +## Infrastructure Components +```bash +# Label namespaces with Ingress resources that should be reachable from Nginx ingress controller with `exposed=true`: +kubectl label namespace goldpinger exposed=true +kubectl label namespace monitoring exposed=true +kubectl label namespace kube-system exposed=true # Required for Hubble-UI. Move Hubble-UI to a dedicated namespaces for production: https://docs.cilium.io/en/stable/gettingstarted/hubble/#enable-the-hubble-ui ("Helm (Standalone install)" tab) + +# Label namespaces with metric endpoints that should be scraped by Prometheus with `metrics=true`: +kubectl label namespace goldpinger metrics=true +kubectl label namespace kube-system metrics=true +kubectl label namespace ingress-nginx metrics=true + +# Kube-system +kubectl apply -f cnp-infra-kube-system.yaml + +# Nginx Ingress Controller +kubectl apply -f cnp-infra-ingress-nginx.yaml + +# Cert-Manager +kubectl apply -f cnp-infra-cert-manager.yaml + +# Kube Prometheus Stack +kubectl apply -f cnp-infra-monitoring-stack.yaml + +# Goldpinger (only in case Goldpinger is deployed on the cluster): +kubectl apply -f cnp-infra-goldpinger.yaml + +# Finally, check for wrongly dropped flows: +hubble observe -t policy-verdict -f --verdict DROPPED +``` + +## Cluster-wide Policies +The goal should be to deploy new user workload namespaces with only a very small set of default policies. Hence, you can leverage CiliumClusterwideNetworkPolicies (CCNPs) to predefine permits of common services like ingress or monitoring to communicate with the new namespace. + +In addition, you can leverage CCNPs to enable Cilium's DNS visibility by applying an egress policy that uses `toPorts[*].rules.dns`. + +```bash +kubectl apply -f ccnp-global-infra.yaml +``` + +## User Workload +As there are already CCNPs matching for everything `spec.endpointSelector: {}`, in both directions (`spec.ingress` and `spec.egress`), newly created (user) namespaces are already in a deny-all state. As a result, even namespace internal traffic is denied until a new `allow-within-namespace` CiliumNetworkPolicy (CNP) is created to allow this traffic: + +```yaml +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-within-namespace + namespace: my-namespace-xy +spec: + description: Allow NS internal traffic, block everything else + endpointSelector: {} + ingress: + - fromEndpoints: + - {} + egress: + - toEndpoints: + - {} +``` + +Have a look at the following template that could be used for new user workload namespaces. + +- `cnp-user-template.yaml`: + - Add the namespace label `exposed: "true"` in case Nginx ingress should be able to serve Ingresses from this namespace. + - Add the namespace label `metrics: "true"` in case Prometheus should be able to scrape metrics endpoints from this namespace. + - Allow all ingress and egress traffic **within** the namespace + - Optional: Additional application specific CNPs to explicitly allow connections to and from namespace-external sources/destinations. + +Check out `demo-app-podinfo.sh` to simulate the deployment of new user workload. + +## Troubleshooting +To troubleshoot connectivity issues or false positive denies, use Hubble UI and especially Hubble CLI. Hubble can either be directly used within a Cilium agent pod (only sees node local traffic) or in an even more powerful way, via the dedicated [Hubble CLI](https://docs.cilium.io/en/stable/gettingstarted/hubble_setup/#install-the-hubble-client). This Hubble CLI then needs to point to the Hubble-Relay service which aggregates the flows from all Cilium agents / nodes. + +```bash +# Temporarily expose the `hubble-relay` ClusterIP service via `kubectl port-forward` (blocking call, separate shell): +kubectl port-forward -n kube-system svc/hubble-relay 4245:443 + +# Check for dropped traffic: +hubble observe -t policy-verdict -f --verdict DROPPED +``` + +Improve your Hubble CLI outputs even further by using additional filtering constraints (issue `hubble observe --help` to see all available options): +- `--ip` / `--to-ip` / `--from-ip` +- `-n` / `--namespace` / `--to-namespace` / `--from-namespace` +- `--port` / `--to-pod` / `--from-pod` +- `--node-name` + +## Sources: +- https://docs.cilium.io/en/stable/gettingstarted/hubble_setup/#install-the-hubble-client +- https://docs.cilium.io/en/stable/gettingstarted/hubble_cli/#hubble-cli \ No newline at end of file diff --git a/netpols/ccnp-global-infra.yaml b/netpols/no-host-policies/ccnp-global-infra.yaml similarity index 86% rename from netpols/ccnp-global-infra.yaml rename to netpols/no-host-policies/ccnp-global-infra.yaml index 4923afa..7d2dbe9 100644 --- a/netpols/ccnp-global-infra.yaml +++ b/netpols/no-host-policies/ccnp-global-infra.yaml @@ -43,7 +43,6 @@ spec: - fields: - type: 8 family: IPv4 -# Heads-up: In case you remove the metrics label again from the NS, check https://github.com/cilium/cilium/issues/27626. --- apiVersion: "cilium.io/v2" kind: CiliumClusterwideNetworkPolicy @@ -60,7 +59,6 @@ spec: - matchLabels: k8s:io.kubernetes.pod.namespace: monitoring k8s:app.kubernetes.io/name: prometheus -# Heads-up: In case you remove the exposed label again from the NS, check https://github.com/cilium/cilium/issues/27626. --- apiVersion: "cilium.io/v2" kind: CiliumClusterwideNetworkPolicy @@ -76,4 +74,4 @@ spec: - fromEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: ingress-nginx - k8s:app.kubernetes.io/name: ingress-nginx \ No newline at end of file + k8s:app.kubernetes.io/name: ingress-nginx diff --git a/netpols/cnp-infra-cert-manager.yaml b/netpols/no-host-policies/cnp-infra-cert-manager.yaml similarity index 92% rename from netpols/cnp-infra-cert-manager.yaml rename to netpols/no-host-policies/cnp-infra-cert-manager.yaml index 86fa999..cf95b9d 100644 --- a/netpols/cnp-infra-cert-manager.yaml +++ b/netpols/no-host-policies/cnp-infra-cert-manager.yaml @@ -33,6 +33,7 @@ spec: - ports: - port: "6443" protocol: TCP +# Only required in case Let's Encrypt (Cluster)Issuer are used: --- apiVersion: "cilium.io/v2" kind: CiliumNetworkPolicy @@ -51,6 +52,7 @@ spec: - ports: - port: "443" protocol: TCP +# Only required in case Let's Encrypt (Cluster)Issuer are used: --- apiVersion: "cilium.io/v2" kind: CiliumNetworkPolicy @@ -87,4 +89,4 @@ spec: toPorts: - ports: - port: "10250" - protocol: TCP \ No newline at end of file + protocol: TCP diff --git a/netpols/no-host-policies/cnp-infra-goldpinger.yaml b/netpols/no-host-policies/cnp-infra-goldpinger.yaml new file mode 100644 index 0000000..e926860 --- /dev/null +++ b/netpols/no-host-policies/cnp-infra-goldpinger.yaml @@ -0,0 +1,38 @@ +# Only in case Goldpinger is deployed on the cluster. +# -- Default rule +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-within-namespace + namespace: goldpinger +spec: + description: Allow NS internal traffic, block everything else + endpointSelector: {} + ingress: + - fromEndpoints: + - {} + egress: + - toEndpoints: + - {} +# Only in case Goldpinger is deployed on the cluster. +# -- Egress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: goldpinger-to-kapi + namespace: goldpinger +spec: + description: Allow Goldpinger to reach KAPI + endpointSelector: + matchLabels: + k8s:app: goldpinger + egress: + - toEntities: + - host + - kube-apiserver + toPorts: + - ports: + - port: "6443" + protocol: TCP diff --git a/netpols/cnp-infra-ingress-nginx.yaml b/netpols/no-host-policies/cnp-infra-ingress-nginx.yaml similarity index 98% rename from netpols/cnp-infra-ingress-nginx.yaml rename to netpols/no-host-policies/cnp-infra-ingress-nginx.yaml index 3e272ea..15346d6 100644 --- a/netpols/cnp-infra-ingress-nginx.yaml +++ b/netpols/no-host-policies/cnp-infra-ingress-nginx.yaml @@ -68,4 +68,4 @@ spec: - port: "80" protocol: TCP - port: "443" - protocol: TCP \ No newline at end of file + protocol: TCP diff --git a/netpols/cnp-infra-kube-system.yaml b/netpols/no-host-policies/cnp-infra-kube-system.yaml similarity index 97% rename from netpols/cnp-infra-kube-system.yaml rename to netpols/no-host-policies/cnp-infra-kube-system.yaml index 5fa80bb..b85afbe 100644 --- a/netpols/cnp-infra-kube-system.yaml +++ b/netpols/no-host-policies/cnp-infra-kube-system.yaml @@ -33,4 +33,4 @@ spec: toPorts: - ports: - port: "53" - protocol: ANY \ No newline at end of file + protocol: ANY diff --git a/netpols/cnp-infra-monitoring-stack.yaml b/netpols/no-host-policies/cnp-infra-monitoring-stack.yaml similarity index 94% rename from netpols/cnp-infra-monitoring-stack.yaml rename to netpols/no-host-policies/cnp-infra-monitoring-stack.yaml index f86b393..a8ded9a 100644 --- a/netpols/cnp-infra-monitoring-stack.yaml +++ b/netpols/no-host-policies/cnp-infra-monitoring-stack.yaml @@ -47,8 +47,8 @@ spec: - key: k8s:app.kubernetes.io/instance operator: In values: - - kube-prometheus-stack-prometheus - - kube-prometheus-stack + - kube-prometheus-stack-prometheus + - kube-prometheus-stack egress: - toEntities: - kube-apiserver @@ -93,6 +93,7 @@ spec: - ports: - port: "6443" protocol: TCP +# Only required if Alertmanager is connected to Slack. --- apiVersion: "cilium.io/v2" kind: CiliumNetworkPolicy @@ -144,12 +145,12 @@ spec: - key: k8s:app.kubernetes.io/instance operator: In values: - - kube-prometheus-stack - - metrics-server + - kube-prometheus-stack + - metrics-server ingress: - fromEntities: - remote-node toPorts: - ports: - port: "10250" - protocol: TCP \ No newline at end of file + protocol: TCP diff --git a/netpols/cnp-user-template.yaml b/netpols/no-host-policies/cnp-user-template.yaml similarity index 96% rename from netpols/cnp-user-template.yaml rename to netpols/no-host-policies/cnp-user-template.yaml index f30da5a..eb7c12d 100644 --- a/netpols/cnp-user-template.yaml +++ b/netpols/no-host-policies/cnp-user-template.yaml @@ -32,4 +32,4 @@ spec: # TBD by app / NS owners # -- Ingress rules -# TBD by app / NS owners \ No newline at end of file +# TBD by app / NS owners diff --git a/netpols/demo-app-podinfo.sh b/netpols/no-host-policies/demo-app-podinfo.sh similarity index 100% rename from netpols/demo-app-podinfo.sh rename to netpols/no-host-policies/demo-app-podinfo.sh diff --git a/netpols/README.md b/netpols/with-host-policies/README.md similarity index 99% rename from netpols/README.md rename to netpols/with-host-policies/README.md index 31dc6a2..55c1943 100644 --- a/netpols/README.md +++ b/netpols/with-host-policies/README.md @@ -1,4 +1,4 @@ -# Demo Network Policies +# Demo Network Policies (with Host Policies) ## Generic Hubble Configuration for Visibility ```bash diff --git a/netpols/with-host-policies/ccnp-global-infra.yaml b/netpols/with-host-policies/ccnp-global-infra.yaml new file mode 100644 index 0000000..7d2dbe9 --- /dev/null +++ b/netpols/with-host-policies/ccnp-global-infra.yaml @@ -0,0 +1,77 @@ +# -- Egress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumClusterwideNetworkPolicy +metadata: + name: global-infra-dns +spec: + description: Allow coredns access from everywhere + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s:k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: ANY + rules: + dns: + - matchPattern: "*" +# -- Ingress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumClusterwideNetworkPolicy +metadata: + name: global-infra-cilium-health +spec: + description: Allow Cilium health checks + endpointSelector: {} + ingress: + # Cilium control plane health endpoint HTTP checks + - fromEntities: + - remote-node + toPorts: + - ports: + - port: "4240" + protocol: ANY + # Cilium control plane health endpoint ICMP checks + - fromEntities: + - remote-node + icmps: + - fields: + - type: 8 + family: IPv4 +--- +apiVersion: "cilium.io/v2" +kind: CiliumClusterwideNetworkPolicy +metadata: + name: global-infra-prometheus +spec: + description: Globally allow connections from Prometheus to labeled namespaces + endpointSelector: + matchLabels: + k8s:io.cilium.k8s.namespace.labels.metrics: "true" + ingress: + # Allow Prometheus to scrape everything + - fromEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: monitoring + k8s:app.kubernetes.io/name: prometheus +--- +apiVersion: "cilium.io/v2" +kind: CiliumClusterwideNetworkPolicy +metadata: + name: global-infra-nginx +spec: + description: Globally allow connections from Nginx ingress controller to labeled namespaces + endpointSelector: + matchLabels: + k8s:io.cilium.k8s.namespace.labels.exposed: "true" + ingress: + # Allow Nginx ingress to reach everything + - fromEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: ingress-nginx + k8s:app.kubernetes.io/name: ingress-nginx diff --git a/netpols/ccnp-host-all.yaml b/netpols/with-host-policies/ccnp-host-all.yaml similarity index 98% rename from netpols/ccnp-host-all.yaml rename to netpols/with-host-policies/ccnp-host-all.yaml index e9508f7..2d5692f 100644 --- a/netpols/ccnp-host-all.yaml +++ b/netpols/with-host-policies/ccnp-host-all.yaml @@ -71,4 +71,4 @@ spec: toPorts: - ports: - port: "10250" - protocol: TCP \ No newline at end of file + protocol: TCP diff --git a/netpols/ccnp-host-cp.yaml b/netpols/with-host-policies/ccnp-host-cp.yaml similarity index 77% rename from netpols/ccnp-host-cp.yaml rename to netpols/with-host-policies/ccnp-host-cp.yaml index 3c987bc..0dc9258 100644 --- a/netpols/ccnp-host-cp.yaml +++ b/netpols/with-host-policies/ccnp-host-cp.yaml @@ -18,10 +18,10 @@ spec: - fromEntities: - kube-apiserver toPorts: - - ports: - # etcd client port - - port: "2379" - protocol: TCP - # etcd peer communication port - - port: "2380" - protocol: TCP \ No newline at end of file + - ports: + # etcd client port + - port: "2379" + protocol: TCP + # etcd peer communication port + - port: "2380" + protocol: TCP diff --git a/netpols/ccnp-host-infra.yaml b/netpols/with-host-policies/ccnp-host-infra.yaml similarity index 95% rename from netpols/ccnp-host-infra.yaml rename to netpols/with-host-policies/ccnp-host-infra.yaml index 18758fa..b4f56be 100644 --- a/netpols/ccnp-host-infra.yaml +++ b/netpols/with-host-policies/ccnp-host-infra.yaml @@ -16,4 +16,4 @@ spec: protocol: TCP # Ingress HTTPS - port: "443" - protocol: TCP \ No newline at end of file + protocol: TCP diff --git a/netpols/with-host-policies/cnp-infra-cert-manager.yaml b/netpols/with-host-policies/cnp-infra-cert-manager.yaml new file mode 100644 index 0000000..cf95b9d --- /dev/null +++ b/netpols/with-host-policies/cnp-infra-cert-manager.yaml @@ -0,0 +1,92 @@ +# -- Default rule +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-within-namespace + namespace: cert-manager +spec: + description: Allow NS internal traffic, block everything else + endpointSelector: {} + ingress: + - fromEndpoints: + - {} + egress: + - toEndpoints: + - {} +# -- Egress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: cert-manager-to-kapi + namespace: cert-manager +spec: + description: Allow all Cert-Manager components to reach KAPI + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/instance: cert-manager + egress: + - toEntities: + - kube-apiserver + toPorts: + - ports: + - port: "6443" + protocol: TCP +# Only required in case Let's Encrypt (Cluster)Issuer are used: +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: cert-manager-to-letsencrypt + namespace: cert-manager +spec: + description: Allow Cert-Manager to reach Let's Encrypt servers + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: cert-manager + egress: + - toFQDNs: + - matchPattern: "*.api.letsencrypt.org" + toPorts: + - ports: + - port: "443" + protocol: TCP +# Only required in case Let's Encrypt (Cluster)Issuer are used: +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: cert-manager-to-own-domain + namespace: cert-manager +spec: + description: Allow Cert-Manager to reach own URLs for the verification check + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: cert-manager + egress: + - toFQDNs: + - matchPattern: "*.philip-netpol-demo.cilium.rocks" + toPorts: + - ports: + - port: "80" + protocol: TCP +# -- Ingress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: kapi-to-cert-manager-webhook + namespace: cert-manager +spec: + description: Allow cluster to reach the Cert-Manager webhook + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: webhook + ingress: + - fromEntities: + - cluster + toPorts: + - ports: + - port: "10250" + protocol: TCP diff --git a/netpols/cnp-infra-goldpinger.yaml b/netpols/with-host-policies/cnp-infra-goldpinger.yaml similarity index 96% rename from netpols/cnp-infra-goldpinger.yaml rename to netpols/with-host-policies/cnp-infra-goldpinger.yaml index d27440a..1128ddf 100644 --- a/netpols/cnp-infra-goldpinger.yaml +++ b/netpols/with-host-policies/cnp-infra-goldpinger.yaml @@ -33,4 +33,4 @@ spec: toPorts: - ports: - port: "6443" - protocol: TCP \ No newline at end of file + protocol: TCP diff --git a/netpols/with-host-policies/cnp-infra-ingress-nginx.yaml b/netpols/with-host-policies/cnp-infra-ingress-nginx.yaml new file mode 100644 index 0000000..15346d6 --- /dev/null +++ b/netpols/with-host-policies/cnp-infra-ingress-nginx.yaml @@ -0,0 +1,71 @@ +# -- Default rule +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-within-namespace + namespace: ingress-nginx +spec: + description: Allow NS internal traffic, block everything else + endpointSelector: {} + ingress: + - fromEndpoints: + - {} + egress: + - toEndpoints: + - {} +# -- Egress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: nginx-ingress-to-kapi + namespace: ingress-nginx +spec: + description: Allow Nginx ingress controller to reach KAPI + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: ingress-nginx + egress: + - toEntities: + - kube-apiserver + toPorts: + - ports: + - port: "6443" + protocol: TCP +--- +apiVersion: "cilium.io/v2" +kind: CiliumClusterwideNetworkPolicy +metadata: + name: nginx-to-services +spec: + description: Allow Nginx ingress controller to access all K8s services of namespaces which expose them + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: ingress-nginx + k8s:io.kubernetes.pod.namespace: ingress-nginx + egress: + - toEndpoints: + - matchLabels: + k8s:io.cilium.k8s.namespace.labels.exposed: "true" +# -- Ingress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: all-to-nginx + namespace: ingress-nginx +spec: + description: Allow accessing Nginx ingress controller from everywhere + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: ingress-nginx + ingress: + - fromEntities: + - all + toPorts: + - ports: + - port: "80" + protocol: TCP + - port: "443" + protocol: TCP diff --git a/netpols/with-host-policies/cnp-infra-kube-system.yaml b/netpols/with-host-policies/cnp-infra-kube-system.yaml new file mode 100644 index 0000000..b85afbe --- /dev/null +++ b/netpols/with-host-policies/cnp-infra-kube-system.yaml @@ -0,0 +1,36 @@ +# -- Default rule +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-to-everywhere + namespace: kube-system +spec: + description: Allow ingress traffic within the namespace and egress to everywhere + endpointSelector: {} + ingress: + - fromEndpoints: + - {} + egress: + - toEntities: + - all +# -- Ingress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-to-coredns + namespace: kube-system +spec: + description: Allow ingress to coredns from everywhere + endpointSelector: + matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s:k8s-app: kube-dns + ingress: + - fromEntities: + - all + toPorts: + - ports: + - port: "53" + protocol: ANY diff --git a/netpols/with-host-policies/cnp-infra-monitoring-stack.yaml b/netpols/with-host-policies/cnp-infra-monitoring-stack.yaml new file mode 100644 index 0000000..a8ded9a --- /dev/null +++ b/netpols/with-host-policies/cnp-infra-monitoring-stack.yaml @@ -0,0 +1,156 @@ +# -- Default rule +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-within-namespace + namespace: monitoring +spec: + description: Allow NS internal traffic, block everything else + endpointSelector: {} + ingress: + - fromEndpoints: + - {} + egress: + - toEndpoints: + - {} +# -- Egress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumClusterwideNetworkPolicy +metadata: + name: prometheus-to-metrics +spec: + description: Allow Prometheus to scrape all infra metric endpoints and the ones of namespaces which expose them + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: prometheus + k8s:io.kubernetes.pod.namespace: monitoring + egress: + - toEntities: + - host + - remote-node + - kube-apiserver + - toEndpoints: + - matchLabels: + k8s:io.cilium.k8s.namespace.labels.metrics: "true" +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: kube-prometheus-stack-to-kapi + namespace: monitoring +spec: + description: Allow kube-prometheus-stack components to reach KAPI + endpointSelector: + matchExpressions: + - key: k8s:app.kubernetes.io/instance + operator: In + values: + - kube-prometheus-stack-prometheus + - kube-prometheus-stack + egress: + - toEntities: + - kube-apiserver + toPorts: + - ports: + - port: "6443" + protocol: TCP +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: metrics-server-to-kubelet-metrics + namespace: monitoring +spec: + description: Allow metrics-server to scrape Kubelet metrics + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: metrics-server + egress: + - toEntities: + - host + - remote-node + toPorts: + - ports: + - port: "10250" + protocol: TCP +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: metrics-server-to-kapi + namespace: monitoring +spec: + description: Allow metrics-server to access KAPI + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: metrics-server + egress: + - toEntities: + - kube-apiserver + toPorts: + - ports: + - port: "6443" + protocol: TCP +# Only required if Alertmanager is connected to Slack. +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: alertmanager-to-slack-webhook + namespace: monitoring +spec: + description: Allow alertmanager to reach Slack webhook + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: alertmanager + egress: + - toFQDNs: + - matchName: "api.slack.com" + - matchName: "hooks.slack.com" + toPorts: + - ports: + - port: "443" + protocol: TCP +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: grafana-to-kapi + namespace: monitoring +spec: + description: Allow Grafana to reach KAPI (to get dashboard and datasource CMs) + endpointSelector: + matchLabels: + k8s:app.kubernetes.io/name: grafana + egress: + - toEntities: + - kube-apiserver + toPorts: + - ports: + - port: "6443" + protocol: TCP +# -- Ingress rules +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: remote-node-to-prometheus + namespace: monitoring +spec: + description: Allow Remote-Node to access Prometheus and metrics-server + endpointSelector: + matchExpressions: + - key: k8s:app.kubernetes.io/instance + operator: In + values: + - kube-prometheus-stack + - metrics-server + ingress: + - fromEntities: + - remote-node + toPorts: + - ports: + - port: "10250" + protocol: TCP diff --git a/netpols/with-host-policies/cnp-user-template.yaml b/netpols/with-host-policies/cnp-user-template.yaml new file mode 100644 index 0000000..eb7c12d --- /dev/null +++ b/netpols/with-host-policies/cnp-user-template.yaml @@ -0,0 +1,35 @@ +# -- User workload NS creation +--- +apiVersion: v1 +kind: Namespace +metadata: + name: my-namespace-xy + # Customized labels + # - Add exposed="true" label in case Nginx should expose something from this NS + # - Add metrics="true" label in case Prometheus should scrape metrics from this NS + labels: + exposed: "true" + metrics: "true" + +# -- Default rule +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-within-namespace + namespace: my-namespace-xy +spec: + description: Allow NS internal traffic, block everything else + endpointSelector: {} + ingress: + - fromEndpoints: + - {} + egress: + - toEndpoints: + - {} + +# -- Egress rules +# TBD by app / NS owners + +# -- Ingress rules +# TBD by app / NS owners diff --git a/netpols/with-host-policies/demo-app-podinfo.sh b/netpols/with-host-policies/demo-app-podinfo.sh new file mode 100755 index 0000000..d30ea9a --- /dev/null +++ b/netpols/with-host-policies/demo-app-podinfo.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# https://github.com/stefanprodan/podinfo/tree/master +# +helm repo add podinfo https://stefanprodan.github.io/podinfo +helm repo update podinfo + +kubectl create namespace podinfo + +# Deploy default CNP +echo "--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-within-namespace + namespace: podinfo +spec: + description: Allow NS internal traffic, block everything else + endpointSelector: {} + ingress: + - fromEndpoints: + - {} + egress: + - toEndpoints: + - {}" | kubectl apply -f- + +kubectl label namespace podinfo exposed=true +kubectl label namespace podinfo metrics=true + +# Install the frontend +cat <