feat: support different kubeconfig paths (CloudNativeSDWAN#26)

This commit brings the ability to use a kubeconfig file that is not the
default one. In case a different one is provided when running as a pod,
egress-watcher will return an error.

Signed-off-by: Elis Lulja <elulja@cisco.com>

pkg/command/run.go

@@ -34,6 +34,7 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/util/homedir"
 )
 
@@ -54,6 +55,11 @@ type Options struct {
 }
 
 func getRunCommand() *cobra.Command {
+	insideCluster := false
+	if _, err := rest.InClusterConfig(); err == nil {
+		insideCluster = true
+	}
+
 	kopts := &kubeConfigOptions{}
 	waitingWindow := sdwan.DefaultWaitingWindow
 	flagOpts := &Options{
@@ -111,6 +117,11 @@ The following controllers are supported:
   You can run the program against vManage by using "vmanage" (or "with-vmanage") as
   first argument.`,
 		PreRunE: func(cmd *cobra.Command, _ []string) error {
+			if insideCluster && kopts.path != "" {
+				return fmt.Errorf("using a different kubeconfig when running " +
+					"inside the cluster is not suppported")
+			}
+
 			if fileSettingsPath != "" {
 				f, err := getSettingsFromFile(fileSettingsPath)
 				if err != nil {
@@ -176,18 +187,18 @@ The following controllers are supported:
 				return fmt.Errorf("no sdwan controller provided")
 			}
 		},
-		Example: "run --kubeconfig /my/kubeconf/.conf --watch-all-service-entries",
+		Example: "run --kubeconfig /my/kubeconf/conf --watch-all-service-entries",
 	}
 
 	// Flags
 	cmd.Flags().StringVar(&kopts.path, "kubeconfig", func() string {
-		if home := homedir.HomeDir(); len(home) > 0 {
+		if home := homedir.HomeDir(); len(home) > 0 && !insideCluster {
 			return path.Join(home, ".kube", "config")
 		}
 
 		return ""
-	}(), "path to the kubeconfig file to use.")
-	cmd.Flags().StringVar(&kopts.context, "context", "", "the context to use.")
+	}(), "path to the kubeconfig file to use. "+
+		"This is only used when running outside of the cluster.")
 	cmd.Flags().BoolVarP(&flagOpts.ServiceEntryController.WatchAllServiceEntries,
 		"watch-all-service-entries", "w", false,
 		"whether to watch all service entries by default.")
@@ -239,7 +250,7 @@ func runWithVmanage(kopts *kubeConfigOptions, opts *Options) error {
 		log.Info().Msg("starting...")
 	}
 
-	mgr, err := controllers.NewManager()
+	mgr, err := controllers.NewManager(kopts.path)
 	if err != nil {
 		return fmt.Errorf("could not get manager: %w", err)
 	}

pkg/controllers/manager.go

@@ -24,11 +24,13 @@ import (
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
 
-func NewManager() (manager.Manager, error) {
+func NewManager(kubeconfigPath string) (manager.Manager, error) {
 	scheme := k8sruntime.NewScheme()
 	if err := clientgoscheme.AddToScheme(scheme); err != nil {
 		return nil, fmt.Errorf("could not add to scheme: %w", err)
@@ -37,7 +39,13 @@ func NewManager() (manager.Manager, error) {
 		return nil, fmt.Errorf("could not add scheme to networking: %w", err)
 	}
 
-	cfg, err := config.GetConfig()
+	cfg, err := func() (*rest.Config, error) {
+		if kubeconfigPath == "" {
+			return config.GetConfig()
+		}
+
+		return clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	}()
 	if err != nil {
 		return nil, fmt.Errorf("could not get config: %w", err)
 	}