feat: use custom socket configurations from v1alpha3 (eraser-dev#930)

Signed-off-by: ashnamehrotra <ashnamehrotra@gmail.com>

Makefile

@@ -162,7 +162,6 @@ collector-dummy-img:
 		test/e2e/test-data
 COLLECTOR_IMAGE_DUMMY=$(COLLECTOR_REPO):dummy
 
-
 vulnerable-img:
 	docker pull $(VULNERABLE_IMG)
 
@@ -178,7 +177,13 @@ non-vulnerable-img:
 		-t ${NON_VULNERABLE_IMG} \
 		--target non-vulnerable .
 
-e2e-test: vulnerable-img eol-img non-vulnerable-img busybox-img collector-dummy-img
+custom-node-v$(KUBERNETES_VERSION):
+	docker build -t custom-node:v$(KUBERNETES_VERSION) \
+		-f test/e2e/test-data/Dockerfile.customNode \
+		--build-arg KUBERNETES_VERSION=${KUBERNETES_VERSION} test/e2e/test-data
+MODIFIED_NODE_IMAGE=custom-node:v$(KUBERNETES_VERSION)
+
+e2e-test: vulnerable-img eol-img non-vulnerable-img busybox-img collector-dummy-img custom-node-v$(KUBERNETES_VERSION)
 	for test in $(E2E_TESTS); do \
 		CGO_ENABLED=0 \
             PROJECT_ABSOLUTE_PATH=$(CURDIR) \
@@ -197,6 +202,7 @@ e2e-test: vulnerable-img eol-img non-vulnerable-img busybox-img collector-dummy-
 			NON_VULNERABLE_IMAGE=${NON_VULNERABLE_IMG} \
 			EOL_IMAGE=${EOL_IMG} \
 			NODE_VERSION=kindest/node:v${KUBERNETES_VERSION} \
+			MODIFIED_NODE_IMAGE=${MODIFIED_NODE_IMAGE} \
 			TEST_LOGDIR=${TEST_LOGDIR} \
 			go test -count=$(TEST_COUNT) -timeout=$(TIMEOUT) $(TESTFLAGS) -tags=e2e -v $$test ; \
 	done

api/unversioned/eraserconfig_types.go

@@ -37,9 +37,10 @@ type (
 )
 
 const (
-	RuntimeContainerd Runtime = "containerd"
-	RuntimeDockerShim Runtime = "dockershim"
-	RuntimeCrio       Runtime = "crio"
+	RuntimeContainerd  Runtime = "containerd"
+	RuntimeDockerShim  Runtime = "dockershim"
+	RuntimeCrio        Runtime = "crio"
+	RuntimeNotProvided Runtime = ""
 
 	ContainerdPath = "/run/containerd/containerd.sock"
 	DockerPath     = "/run/dockershim.sock"
@@ -124,6 +125,14 @@ func (r *RuntimeSpec) UnmarshalJSON(b []byte) error {
 		}
 
 		*r = converted
+	case RuntimeNotProvided:
+		if rs.Address != "" {
+			return fmt.Errorf("runtime name must be provided with address")
+		}
+
+		// if empty name and address, use containerd as default
+		r.Name = RuntimeContainerd
+		r.Address = fmt.Sprintf("unix://%s", ContainerdPath)
 	default:
 		return fmt.Errorf("invalid runtime: valid names are %s, %s, %s", RuntimeContainerd, RuntimeDockerShim, RuntimeCrio)
 	}

api/v1alpha1/custom_conversions.go

@@ -13,7 +13,13 @@ func Convert_v1alpha1_ManagerConfig_To_unversioned_ManagerConfig(in *ManagerConf
 //nolint:revive
 func manualConvert_v1alpha1_Runtime_To_unversioned_RuntimeSpec(in *Runtime, out *unversioned.RuntimeSpec, _ conversion.Scope) error {
 	out.Name = unversioned.Runtime(string(*in))
-	out.Address = ""
+
+	rs, err := unversioned.ConvertRuntimeToRuntimeSpec(out.Name)
+	if err != nil {
+		return err
+	}
+	out.Address = rs.Address
+
 	return nil
 }
 

api/v1alpha2/custom_conversions.go

@@ -13,7 +13,13 @@ func Convert_v1alpha2_ManagerConfig_To_unversioned_ManagerConfig(in *ManagerConf
 //nolint:revive
 func manualConvert_v1alpha2_Runtime_To_unversioned_RuntimeSpec(in *Runtime, out *unversioned.RuntimeSpec, _ conversion.Scope) error {
 	out.Name = unversioned.Runtime(string(*in))
-	out.Address = ""
+
+	rs, err := unversioned.ConvertRuntimeToRuntimeSpec(out.Name)
+	if err != nil {
+		return err
+	}
+	out.Address = rs.Address
+
 	return nil
 }
 

api/v1alpha3/eraserconfig_types.go

@@ -37,9 +37,10 @@ type (
 )
 
 const (
-	RuntimeContainerd Runtime = "containerd"
-	RuntimeDockerShim Runtime = "dockershim"
-	RuntimeCrio       Runtime = "crio"
+	RuntimeContainerd  Runtime = "containerd"
+	RuntimeDockerShim  Runtime = "dockershim"
+	RuntimeCrio        Runtime = "crio"
+	RuntimeNotProvided Runtime = ""
 
 	ContainerdPath = "/run/containerd/containerd.sock"
 	DockerPath     = "/run/dockershim.sock"
@@ -124,6 +125,14 @@ func (r *RuntimeSpec) UnmarshalJSON(b []byte) error {
 		}
 
 		*r = converted
+	case RuntimeNotProvided:
+		if rs.Address != "" {
+			return fmt.Errorf("runtime name must be provided with address")
+		}
+
+		// if empty name and address, use containerd as default
+		r.Name = RuntimeContainerd
+		r.Address = fmt.Sprintf("unix://%s", ContainerdPath)
 	default:
 		return fmt.Errorf("invalid runtime: valid names are %s, %s, %s", RuntimeContainerd, RuntimeDockerShim, RuntimeCrio)
 	}

controllers/imagecollector/imagecollector_controller.go

@@ -438,6 +438,10 @@ func (r *Reconciler) createImageJob(ctx context.Context) (ctrl.Result, error) {
 					Name:  "OTEL_SERVICE_NAME",
 					Value: "trivy-scanner",
 				},
+				{
+					Name:  "ERASER_RUNTIME_NAME",
+					Value: string(mgrCfg.Runtime.Name),
+				},
 			},
 		}
 		jobTemplate.Spec.Containers = append(jobTemplate.Spec.Containers, scannerContainer)

controllers/imagejob/imagejob_controller.go

@@ -16,6 +16,7 @@ package imagejob
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
 	"time"
@@ -42,6 +43,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 	"sigs.k8s.io/kind/pkg/errors"
 
+	"github.com/eraser-dev/eraser/api/unversioned"
 	"github.com/eraser-dev/eraser/api/unversioned/config"
 	eraserv1 "github.com/eraser-dev/eraser/api/v1"
 	controllerUtils "github.com/eraser-dev/eraser/controllers/util"
@@ -398,7 +400,7 @@ func (r *Reconciler) handleNewJob(ctx context.Context, imageJob *eraserv1.ImageJ
 		}
 
 		log := log.WithValues("node", nodeNames[i])
-		podSpec, err := copyAndFillTemplateSpec(&podSpecTemplate, env, currNode)
+		podSpec, err := copyAndFillTemplateSpec(&podSpecTemplate, env, currNode, &eraserConfig.Manager.Runtime)
 		if err != nil {
 			return err
 		}
@@ -587,36 +589,31 @@ nodes:
 	return newNodeNames, skipped, nil
 }
 
-func copyAndFillTemplateSpec(templateSpecTemplate *corev1.PodSpec, env []corev1.EnvVar, node *corev1.Node) (*corev1.PodSpec, error) {
+func copyAndFillTemplateSpec(templateSpecTemplate *corev1.PodSpec, env []corev1.EnvVar, node *corev1.Node, runtimeSpec *unversioned.RuntimeSpec) (*corev1.PodSpec, error) {
 	nodeName := node.Name
-	runtime := node.Status.NodeInfo.ContainerRuntimeVersion
-	runtimeName := strings.Split(runtime, ":")[0]
 
-	mountPath, ok := eraserUtils.RuntimeSocketPathMap[runtimeName]
-	if !ok {
-		return nil, fmt.Errorf("incompatible runtime on node")
+	u, err := url.Parse(runtimeSpec.Address)
+	if err != nil {
+		return nil, err
 	}
 
-	args := []string{"--runtime=" + runtimeName}
 	volumes := []corev1.Volume{
-		{Name: runtimeName + "-sock-volume", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: mountPath}}},
+		{Name: "runtime-sock-volume", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: u.Path}}},
 	}
 
 	volumeMounts := []corev1.VolumeMount{
-		{MountPath: mountPath, Name: runtimeName + "-sock-volume"},
+		{MountPath: controllerUtils.CRIPath, Name: "runtime-sock-volume"},
 	}
 
 	templateSpec := templateSpecTemplate.DeepCopy()
 	templateSpec.Tolerations = defaultTolerations
 
 	eraserImg := &templateSpec.Containers[0]
-	eraserImg.Args = append(eraserImg.Args, args...)
 	eraserImg.VolumeMounts = append(eraserImg.VolumeMounts, volumeMounts...)
 	eraserImg.Env = append(eraserImg.Env, env...)
 
 	if len(templateSpec.Containers) > 1 {
 		collectorImg := &templateSpec.Containers[1]
-		collectorImg.Args = append(collectorImg.Args, args...)
 		collectorImg.VolumeMounts = append(collectorImg.VolumeMounts, volumeMounts...)
 		collectorImg.Env = append(collectorImg.Env, env...)
 	}
@@ -625,10 +622,6 @@ func copyAndFillTemplateSpec(templateSpecTemplate *corev1.PodSpec, env []corev1.
 		scannerImg := &templateSpec.Containers[2]
 		scannerImg.VolumeMounts = append(scannerImg.VolumeMounts, volumeMounts...)
 		scannerImg.Env = append(scannerImg.Env,
-			corev1.EnvVar{
-				Name:  eraserUtils.EnvEraserContainerRuntime,
-				Value: runtimeName,
-			},
 			corev1.EnvVar{
 				Name:  controllerUtils.EnvVarContainerdNamespaceKey,
 				Value: controllerUtils.EnvVarContainerdNamespaceValue,

controllers/util/util.go

@@ -31,6 +31,7 @@ const (
 
 	EnvVarContainerdNamespaceKey   = "CONTAINERD_NAMESPACE"
 	EnvVarContainerdNamespaceValue = "k8s.io"
+	CRIPath                        = "/run/cri/cri.sock"
 )
 
 func NeverOnCreate(_ event.CreateEvent) bool {

docs/docs/customization.md

@@ -87,7 +87,9 @@ these options, see the [table](#detailed-options).
 
 ```yaml
 manager:
-  runtime: containerd
+  runtime:
+    name: containerd
+    address: unix:///run/containerd/containerd.sock
   otlpEndpoint: "" # empty string disables OpenTelemetry
   logLevel: info
   profile:
@@ -192,7 +194,8 @@ timeout:
 
 | Option | Description | Default |
 | --- | --- | --- |
-| manager.runtime | The runtime to use for the manager's containers. Must be one of containerd, crio, or dockershim. It is assumed that your nodes are all using the same runtime, and there is currently no way to configure multiple runtimes. | containerd |
+| manager.runtime.name | The runtime to use for the manager's containers. Must be one of containerd, crio, or dockershim. It is assumed that your nodes are all using the same runtime, and there is currently no way to configure multiple runtimes. | containerd |
+| manager.runtime.address | The runtime socket address to use for the containers. Can provide a custom address for containerd and dockershim runtimes, but not for crio due to Trivy restrictions. | unix:///run/containerd/containerd.sock |
 | manager.otlpEndpoint | The endpoint to send OpenTelemetry data to. If empty, data will not be sent. | "" |
 | manager.logLevel | The log level for the manager's containers. Must be one of debug, info, warn, error, dpanic, panic, or fatal. | info |
 | manager.scheduling.repeatInterval | Use only when collector ando/or scanner are enabled. This is like a cron job, and will spawn an _ImageJob_ at the interval provided. | 24h |

pkg/collector/collector.go

@@ -19,7 +19,6 @@ import (
 )
 
 var (
-	runtimePtr    = flag.String("runtime", "containerd", "container runtime")
 	enableProfile = flag.Bool("enable-pprof", false, "enable pprof profiling")
 	profilePort   = flag.Int("pprof-port", 6060, "port for pprof profiling. defaulted to 6060 if unspecified")
 	scanDisabled  = flag.Bool("scan-disabled", false, "boolean for if scanner container is disabled")
@@ -49,13 +48,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	socketPath, found := util.RuntimeSocketPathMap[*runtimePtr]
-	if !found {
-		log.Error(fmt.Errorf("unsupported runtime"), "runtime", *runtimePtr)
-		os.Exit(1)
-	}
-
-	client, err := cri.NewCollectorClient(socketPath)
+	client, err := cri.NewCollectorClient(util.CRIPath)
 	if err != nil {
 		log.Error(err, "failed to get image client")
 		os.Exit(1)

pkg/remover/remover.go

@@ -26,7 +26,6 @@ import (
 )
 
 var (
-	runtimePtr    = flag.String("runtime", "containerd", "container runtime")
 	imageListPtr  = flag.String("imagelist", "", "name of ImageList")
 	enableProfile = flag.Bool("enable-pprof", false, "enable pprof profiling")
 	profilePort   = flag.Int("pprof-port", 6060, "port for pprof profiling. defaulted to 6060 if unspecified")
@@ -60,13 +59,7 @@ func main() {
 		os.Exit(generalErr)
 	}
 
-	socketPath, found := util.RuntimeSocketPathMap[*runtimePtr]
-	if !found {
-		log.Error(fmt.Errorf("unsupported runtime"), "runtime", *runtimePtr)
-		os.Exit(generalErr)
-	}
-
-	client, err := cri.NewRemoverClient(socketPath)
+	client, err := cri.NewRemoverClient(util.CRIPath)
 	if err != nil {
 		log.Error(err, "failed to get image client")
 		os.Exit(generalErr)

pkg/scanners/trivy/trivy.go

@@ -160,12 +160,11 @@ func initScanner(userConfig *Config) (Scanner, error) {
 	sugar := logger.Sugar()
 	trivylogger.Logger = sugar
 
-	runtime := os.Getenv(utils.EnvEraserContainerRuntime)
-	if runtime == "" {
-		runtime = utils.RuntimeContainerd
+	userConfig.Runtime = unversioned.RuntimeSpec{
+		Name:    unversioned.Runtime(os.Getenv(utils.EnvEraserRuntimeName)),
+		Address: utils.CRIPath,
 	}
 
-	userConfig.Runtime = runtime
 	totalTimeout := time.Duration(userConfig.Timeout.Total)
 	timer := time.NewTimer(totalTimeout)