Merge pull request #202 from dciangot/doc_split_fix

Finalize splitting of the plugin repositories. Doc updated with working demo

cmd/installer/templates/deployment.yaml

@@ -25,25 +25,20 @@ spec:
         env:
         - name: NODENAME
           value: {{.VKName}} 
-        - name: CONFIGPATH
-          value: /etc/interlink/interlink-cfg.json
         - name: KUBELET_PORT
           value: "10250"
         - name: POD_IP
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
-        - name: INTERLINKCONFIGPATH
+        - name: CONFIGPATH
           value: "/etc/interlink/InterLinkConfig.yaml"
         - name: VKTOKENFILE
           value: "/opt/interlink/token"
         volumeMounts:
         - name: config
           mountPath: /etc/interlink/InterLinkConfig.yaml
           subPath: InterLinkConfig.yaml
-        - name: config
-          mountPath: /etc/interlink/kubeconfig
-          subPath: kubeconfig
         - name: token
           mountPath: /opt/interlink
         resources:
@@ -91,5 +86,5 @@ spec:
           name: {{.VKName}}-config
       - name: token
         hostPath:
-          path: /tmp/token
+          path: /tmp
           type: Directory

cmd/virtual-kubelet/main.go

@@ -92,7 +92,7 @@ type Opts struct {
 }
 
 // NewOpts returns an Opts struct with the default values set.
-func NewOpts(nodename string, configpath string, config commonIL.VirtualKubeletConfig) *Opts {
+func NewOpts(nodename string, configpath string) error {
 
 	if nodename == "" {
 		nodename = os.Getenv("NODENAME")
@@ -102,12 +102,7 @@ func NewOpts(nodename string, configpath string, config commonIL.VirtualKubeletC
 		configpath = os.Getenv("CONFIGPATH")
 	}
 
-	return &Opts{
-		ConfigPath: configpath,
-		NodeName:   nodename,
-		Verbose:    config.VerboseLogging,
-		ErrorsOnly: config.ErrorsOnlyLogging,
-	}
+	return nil
 }
 
 func initProvider() (func(context.Context) error, error) {
@@ -169,12 +164,16 @@ func main() {
 	defer cancel()
 	nodename := flag.String("nodename", "", "The name of the node")
 	configpath := flag.String("configpath", "", "Path to the VK config")
-	flag.Parse()
+  flag.Parse()
+	err := NewOpts(*nodename, *configpath)
+	if err != nil {
+		panic(err)
+	}
+
 	interLinkConfig, err := commonIL.LoadConfig(*configpath, *nodename, ctx)
 	if err != nil {
 		panic(err)
 	}
-	opts := NewOpts(*nodename, *configpath, interLinkConfig)
 
 	logger := logrus.StandardLogger()
 	if interLinkConfig.VerboseLogging {

docs/docs/tutorial-admins/01-deploy-interlink.mdx

@@ -23,9 +23,7 @@ In this tutorial, we suppose the remote VM fully owned (not shared) by the user
 ## Create an OAuth GitHub app
 
 :::warning
-
 In this tutoria GitHub tokens are just an example of authentication mechanism, any OpenID compliant identity provider is also supported with the very same deployment script, see [examples here](./04-oidc-IAM.md).
-
 :::
 
 As a first step, you need to create a GitHub OAuth application to allow interLink to make authentication between your Kubernetes cluster and the remote endpoint.
@@ -73,14 +71,15 @@ Now it's all set for the next steps.
 You can download the interLink **installer CLI** for your OS and processor architecture from the [release page](https://github.com/interTwin-eu/interLink/releases), looking for the binaries starting with `interlink-install`. For instance, if on a `Linux` platform with `x86_64` processor:
 
 ```bash
-wget -O interlink-install https://github.com/interTwin-eu/interLink/releases/download/0.1.2/interlink-install_Linux_x86_64
+export VERSION=0.2.3-pre6
+wget -O interlink-install https://github.com/interTwin-eu/interLink/releases/download/$VERSION/interlink-install_Linux_x86_64
 chmod +x interlink-install
 ```
 
 The CLI offers a utility option to initiate an empty config file for the installation at `$HOME/.interlink.yaml`:
 
 ```bash
-interlink-install --init
+./interlink-install --init
 ```
 
 You are now ready to go ahead and edit the produced file with all the setup information.
@@ -133,7 +132,7 @@ You are ready now to go ahead generating the needed manifests and script for the
 Generate the manifests and the automatic interlink installation script with:
 
 ```bash
-interlink-install 
+./interlink-install 
 ```
 
 follow the instruction to authenticate with the device code flow and, if everything went well, you should get an output like the following:
@@ -210,16 +209,115 @@ To stop or restart the components you can use the dedicated commands:
 In alterative you can start an already supported one.
 
 
+### Remote SLURM job submission
+
+:::warning
+For more details on configuration and installation please refer to the SLURM plugin repository: [github.com/interTwin-eu/interlink-docker-plugin](https://github.com/interTwin-eu/interlink-docker-plugin) 
+:::
+
+#### Requirements
+
+- a slurm CLI available on the remote host and configured to interact with the computing cluster
+- a sharedFS with all the worker nodes
+  - an experimental feature is available for cases in which this is not possible
+
+#### Configuration
+
+Create a config file `.interlink/config/slurm.yaml`:
+
+```yaml
+# Sidecar local endpoint
+SidecarPort: "4000"
+SidecarURL: "http://localhost"
+
+# Prefix for every plugin slurm command
+CommandPrefix: ""
+
+# Use sharedFS for configmap and secrets
+ExportPodData: true
+
+# Directory where all the core interlink logs and jobs will be tracked
+DataRootFolder: "/home/civo/.interlink/"
+
+# The endpoint for the slurm plugin daemon
+InterlinkURL: "http://localhost"
+InterlinkPort: "30080"
+
+# Absolute path to slurm and bash command
+SbatchPath: "/usr/bin/sbatch"
+ScancelPath: "/usr/bin/scancel"
+SqueuePath: "/usr/bin/squeue"
+BashPath: "/bin/bash"
+```
+
+:::danger
+Before going ahead, put the correct DataRootFolder in the example above! Don't forget the `/` at the end!
+:::
+
+#### Systemd installation
+
+:::warning
+To get the latest version of the plugin, please visit the [release](https://github.com/interTwin-eu/interlink-slurm-plugin/releases) page.
+:::
+
+Download the latest release with:
+
+```bash
+VERSION=0.2.2
+wget -O $HOME/.interlink/bin/slurm-plugin https://github.com/interTwin-eu/interlink-slurm-plugin/releases/download/$VERSION/interlink-sidecar-slurm_Linux_x86_64
+chmod +x $HOME/.interlink/bin/slurm-plugin
+```
+
+Now you can create a systemd service on the user space with the following:
+
+```bash
+mkdir -p $HOME/.config/systemd/user
+
+cat <<EOF > $HOME/.config/systemd/user/slurm-plugin.service
+[Unit]
+Description=This Unit is needed to automatically start the SLURM sidecar at system startup
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=$HOME/.interlink/bin/slurm-plugin
+Environment="INTERLINKCONFIGPATH=$HOME/.interlink/config/InterLinkConfig.yaml"
+Environment="SHARED_FS=true"
+StandardOutput=file:$HOME/.interlink/logs/plugin.log
+StandardError=file:$HOME/.interlink/logs/plugin.log
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl --user daemon-reload
+systemctl --user enable slurm-plugin.service
+```
+
+An eventually starting and monitoring with `start` and `status`:
+
+```
+systemctl --user start slurm-plugin.service
+systemctl --user status slurm-plugin.service
+```
+
+Logs will be stored at `$HOME/.interlink/logs/plugin.log`.
+
 ### Remote docker execution
 
 :::warning
-Coming soon...
+An mantained plugin will come soon...
+In the meantime you can take a look at the ["developing a plugin"](./02-develop-a-plugin.md) example.
 :::
 
-### Remote SLURM job submission
+- [Docker plugin repository](https://github.com/interTwin-eu/interlink-docker-plugin)
+
+### Submit pods to HTcondor or ARC batch systems
 
 :::warning
-Coming soon...
+Coming soon
 :::
-
+
+- [HTCondor plugin repository](https://github.com/interTwin-eu/interlink-htcondor-plugin)
+- [ARC plugin repository](https://github.com/interTwin-eu/interlink-arc-plugin)
 

docs/docs/tutorial-admins/02-develop-a-plugin.md

@@ -34,7 +34,7 @@ Then you are ready to install the python SDK with:
 
 ```bash
 #export VERSION=X.X.X
-#pip install "uvicorn[standard]" "git+https://github.com/interTwin-eu/interLink.git@${VERSION}#egg=interlink&subdirectory=examples/sidecar/templates/python"
+#pip install "uvicorn[standard]" "git+https://github.com/interTwin-eu/interLink.git@${VERSION}#egg=interlink&subdirectory=example"
 
 # Or download the latest one with
 pip install "uvicorn[standard]" "git+https://github.com/interTwin-eu/interLink.git#egg=interlink&subdirectory=example"
@@ -48,7 +48,7 @@ In the next section we are going to leverage the provider class of SDK to create
 
 The [provider class](https://github.com/interTwin-eu/interLink/blob/main/example/interlink/provider.py) is a FastAPI interface that aims to isolate the developers from all the API provisioning boiler plate.
 
-In fact we are going to need only the creation of a derived class implementing the [interLink core methods](https://github.com/interTwin-eu/interLink/blob/main/example/provider.py#L14-L24),
+In fact we are going to need only the creation of a derived class implementing the [interLink core methods](https://github.com/interTwin-eu/interLink/blob/main/example/interlink/provider.py#L14-L24),
 and making use of in [request and response API specification](https://github.com/interTwin-eu/interLink/blob/main/example/interlink/spec.py) to create our own container lifecycle management plugin.
 
 
@@ -307,7 +307,7 @@ When receiving the LogRequest, there are many log options to satisfy, in any cas
 
 ### A more advanced example
 
-If you are interested in a more advanced example, please refer the [full example](https://github.com/interTwin-eu/interLink/blob/main/example/main.py)
+If you are interested in a more advanced example, please refer the [full example](https://github.com/interTwin-eu/interLink/blob/main/example/provider_demo.py)
 for supporting configMap and secret volumes.
 
 ## Let's test is out

docs/docs/tutorial-admins/04-oidc-IAM.md

@@ -8,8 +8,28 @@ In alternative of the Github authentication flow, we support any OpenID complian
 
 ## EGI Check-in
 
-:::warning
-TBD
+If you have an account for [EGI check-in](https://aai.egi.eu), you should be able to set it for authenticating the virtual kubelet with the interLink remote components with the following piece of configuration to be passed to the [installation script](./01-deploy-interlink.mdx).
+
+```yaml
+oauth:
+  provider: oidc
+  issuer: https://aai.egi.eu/auth/realms/egi
+  scopes:
+    - "openid"
+    - "email"
+    - "offline_access"
+    - "profile"
+  audience: interlink
+  group_claim: email
+  group: "YOUR EMAIL HERE"
+  token_url: "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token"
+  device_code_url: "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/auth/device"
+  client_id: "oidc-agent"
+  client_secret: ""
+```
+
+:::danger
+Remember to put your email in the group field!
 :::
 
 ## Indigo IAM

docs/docs/tutorial-users/01-quick-start.md

@@ -19,21 +19,21 @@ __N.B.__ in the demo the oauth2 proxy authN/Z is disabled. DO NOT USE THIS IN PR
 - Clone interlink repo:
 
 ```bash
-git clone --branch 0.2.3-pre2 https://github.com/interTwin-eu/interLink.git 
+git clone https://github.com/interTwin-eu/interLink.git 
 ```
 
 ## Connect a remote machine with Docker 
 
 Move to example location:
 
 ```bash
-cd interLink/examples/interlink-docker
+cd interLink/example/interlink-docker
 ```
 
 ### Setup Kubernetes cluster
 
 ```bash
-minikube start --kubernetes-version=1.26.10
+minikube start --kubernetes-version=1.27.1
 ```
 
 ### Deploy Interlink
@@ -45,7 +45,11 @@ You need to provide the interLink IP address that should be reachable from the k
 ```bash
 export INTERLINK_IP_ADDRESS=XXX.XX.X.XXX
 
-sed -i 's/InterlinkURL:.*/InterlinkURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g'  vk/InterLinkConfig.yaml | sed -i 's/SidecarURL:.*/SidecarURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' vk/InterLinkConfig.yaml
+sed -i 's/InterlinkURL:.*/InterlinkURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g'  vk/InterLinkConfig.yaml
+
+sed -i 's/InterlinkURL:.*/InterlinkURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g'  interlink/InterLinkConfig.yaml | sed -i 's/SidecarURL:.*/SidecarURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' interlink/InterLinkConfig.yaml
+
+sed -i 's/InterlinkURL:.*/InterlinkURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g'  interlink/sidecarConfig.yaml | sed -i 's/SidecarURL:.*/SidecarURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' interlink/sidecarConfig.yaml
 ```
 
 #### Deploy virtualKubelet

example/interlink-docker/interlink/InterLinkConfig.yaml

@@ -0,0 +1,9 @@
+VKTokenFile: "$HOME/interLink/token"
+InterlinkURL: "http://XXX.XXX.XXX.XXX"
+SidecarURL: "http://docker-sidecar"
+InterlinkPort: "3000"
+SidecarPort: "4000"
+ExportPodData: true
+DataRootFolder: ".local/interlink/jobs/"
+VerboseLogging: true
+ErrorsOnlyLogging: false

example/interlink-docker/interlink/docker-compose.yaml

@@ -0,0 +1,46 @@
+version: '3.7'
+services:
+  interlink:
+    build:
+      context: ../../../
+      dockerfile: docker/Dockerfile.interlink
+    restart: always
+    #network_mode: "host"
+    ports:
+    - 3000:3000
+    volumes:
+    - type: bind
+      source: ./
+      target: /etc/interlink
+    environment:
+    - INTERLINKCONFIGPATH=/etc/interlink/InterLinkConfig.yaml
+    # healthcheck:
+    #   test: ["CMD", "/check.sh"]
+    #   interval: 10s
+    #   timeout: 10s
+    #   retries: 3
+    #   start_period: 5s
+  docker-sidecar:
+    image: ghcr.io/intertwin-eu/interlink-sidecar-docker:0.0.2-pre1
+    restart: always
+    privileged: true
+    cap_add:
+    - SYS_ADMIN
+    #network_mode: "host"
+    ports:
+    - 4000:4000
+    environment:
+    - INTERLINKCONFIGPATH=/etc/interlink/sidecarConfig.yaml
+    volumes:
+    - type: bind
+      source: ./
+      target: /etc/interlink
+    - type: bind
+      source: /var/run/docker.sock 
+      target: /var/run/docker.sock 
+    # healthcheck:
+    #   test: ["CMD", "/check.sh"]
+    #   interval: 10s
+    #   timeout: 10s
+    #   retries: 3
+    #   start_period: 5s