diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index cd32d39..239a01d 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -36,6 +36,9 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { for _, data := range req { + podUID := string(data.Pod.UID) + podNamespace := string(data.Pod.Namespace) + pathsOfVolumes := make(map[string]string) for _, volume := range data.Pod.Spec.Volumes { @@ -61,6 +64,8 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { for _, container := range data.Pod.Spec.Containers { + containerName := podNamespace + "-" + podUID + "-" + container.Name + var isGpuRequested bool = false var additionalGpuArgs []string @@ -72,7 +77,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { isGpuRequested = true - log.G(h.Ctx).Info("Container " + container.Name + " is requesting " + val.String() + " GPU") + log.G(h.Ctx).Info("Container " + containerName + " is requesting " + val.String() + " GPU") numGpusRequestedInt := int(numGpusRequested) _, err := h.GpuManager.GetAvailableGPUs(numGpusRequestedInt) @@ -82,7 +87,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } - gpuSpecs, err := h.GpuManager.GetAndAssignAvailableGPUs(numGpusRequestedInt, container.Name) + gpuSpecs, err := h.GpuManager.GetAndAssignAvailableGPUs(numGpusRequestedInt, containerName) if err != nil { HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) return @@ -100,11 +105,9 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { additionalGpuArgs = append(additionalGpuArgs, "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES="+gpuUUIDs) } else { - log.G(h.Ctx).Info("Container " + container.Name + " is not requesting a GPU") + log.G(h.Ctx).Info("Container " + containerName + " is not requesting a GPU") } - log.G(h.Ctx).Info("- Creating container " + container.Name) - var envVars string = "" // add environment variables to the docker command for _, envVar := range container.Env { @@ -138,9 +141,9 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { } } - log.G(h.Ctx).Info("- Creating container " + container.Name) + log.G(h.Ctx).Info("- Creating container " + containerName) - cmd := []string{"run", "-d", "--name", container.Name} + cmd := []string{"run", "-d", "--name", containerName} cmd = append(cmd, envVars) @@ -195,34 +198,34 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { } if execReturn.Stdout == "" { - eval := "Conflict. The container name \"/" + container.Name + "\" is already in use" + eval := "Conflict. The container name \"/" + containerName + "\" is already in use" if strings.Contains(execReturn.Stderr, eval) { - log.G(h.Ctx).Warning("Container named " + container.Name + " already exists. Skipping its creation.") + log.G(h.Ctx).Warning("Container named " + containerName + " already exists. Skipping its creation.") } else { - log.G(h.Ctx).Error("Unable to create container " + container.Name + " : " + execReturn.Stderr) + log.G(h.Ctx).Error("Unable to create container " + containerName + " : " + execReturn.Stderr) HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) return } } else { - log.G(h.Ctx).Info("-- Created container " + container.Name) + log.G(h.Ctx).Info("-- Created container " + containerName) } shell = exec.ExecTask{ Command: "docker", - Args: []string{"ps", "-aqf", "name=^" + container.Name + "$"}, + Args: []string{"ps", "-aqf", "name=^" + containerName + "$"}, Shell: true, } execReturn, err = shell.Execute() execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") if execReturn.Stderr != "" { - log.G(h.Ctx).Error("Failed to retrieve " + container.Name + " ID : " + execReturn.Stderr) + log.G(h.Ctx).Error("Failed to retrieve " + containerName + " ID : " + execReturn.Stderr) HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) return } else if execReturn.Stdout == "" { log.G(h.Ctx).Error("Container name not found. Maybe creation failed?") } else { - log.G(h.Ctx).Debug("-- Retrieved " + container.Name + " ID: " + execReturn.Stdout) + log.G(h.Ctx).Debug("-- Retrieved " + containerName + " ID: " + execReturn.Stdout) } } } diff --git a/pkg/docker/Delete.go b/pkg/docker/Delete.go index 232e018..09feb83 100644 --- a/pkg/docker/Delete.go +++ b/pkg/docker/Delete.go @@ -37,11 +37,18 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { return } + podUID := string(pod.UID) + podNamespace := string(pod.Namespace) + + for _, container := range pod.Spec.Containers { - log.G(h.Ctx).Debug("- Deleting container " + container.Name) + + containerName := podNamespace + "-" + podUID + "-" + container.Name + + log.G(h.Ctx).Debug("- Deleting container " + containerName) // added a timeout to the stop container command - cmd := []string{"stop", "-t", "10", container.Name} + cmd := []string{"stop", "-t", "10", containerName} shell := exec.ExecTask{ Command: "docker", Args: cmd, @@ -51,9 +58,9 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { if execReturn.Stderr != "" { if strings.Contains(execReturn.Stderr, "No such container") { - log.G(h.Ctx).Debug("-- Unable to find container " + container.Name + ". Probably already removed? Skipping its removal") + log.G(h.Ctx).Debug("-- Unable to find container " + containerName + ". Probably already removed? Skipping its removal") } else { - log.G(h.Ctx).Error("-- Error stopping container " + container.Name + ". Skipping its removal") + log.G(h.Ctx).Error("-- Error stopping container " + containerName + ". Skipping its removal") statusCode = http.StatusInternalServerError w.WriteHeader(statusCode) w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs")) @@ -73,18 +80,18 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") if execReturn.Stderr != "" { - log.G(h.Ctx).Error("-- Error deleting container " + container.Name) + log.G(h.Ctx).Error("-- Error deleting container " + containerName) statusCode = http.StatusInternalServerError w.WriteHeader(statusCode) w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs")) return } else { - log.G(h.Ctx).Info("- Deleted container " + container.Name) + log.G(h.Ctx).Info("- Deleted container " + containerName) } } // check if the container has GPU devices attacched using the GpuManager and release them - h.GpuManager.Release(container.Name) + h.GpuManager.Release(containerName) os.RemoveAll(h.Config.DataRootFolder + pod.Namespace + "-" + string(pod.UID)) } diff --git a/pkg/docker/GetLogs.go b/pkg/docker/GetLogs.go index 85a6c44..15d6bd1 100644 --- a/pkg/docker/GetLogs.go +++ b/pkg/docker/GetLogs.go @@ -44,13 +44,16 @@ func (h *SidecarHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request) return } - //req = test + podUID := string(req.PodUID) + podNamespace := string(req.Namespace) + + containerName := podNamespace + "-" + podUID + "-" + req.ContainerName var cmd *OSexec.Cmd if req.Opts.Timestamps { - cmd = OSexec.Command("docker", "logs", "-t", req.ContainerName) + cmd = OSexec.Command("docker", "logs", "-t", containerName) } else { - cmd = OSexec.Command("docker", "logs", req.ContainerName) + cmd = OSexec.Command("docker", "logs", containerName) } output, err := cmd.CombinedOutput() diff --git a/pkg/docker/Status.go b/pkg/docker/Status.go index c6f203d..fd883a5 100644 --- a/pkg/docker/Status.go +++ b/pkg/docker/Status.go @@ -39,10 +39,16 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { } for i, pod := range req { - resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodUID: string(pod.UID), PodNamespace: pod.Namespace}) + + podUID := string(pod.UID) + podNamespace := string(pod.Namespace) + + resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodUID: podUID, PodNamespace: podNamespace}) for _, container := range pod.Spec.Containers { - log.G(h.Ctx).Debug("- Getting status for container " + container.Name) - cmd := []string{"ps -af name=^" + container.Name + "$ --format \"{{.Status}}\""} + containerName := podNamespace + "-" + podUID + "-" + container.Name + + log.G(h.Ctx).Debug("- Getting status for container " + containerName) + cmd := []string{"ps -af name=^" + containerName + "$ --format \"{{.Status}}\""} shell := exec.ExecTask{ Command: "docker", @@ -63,19 +69,19 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { // TODO: why first container? if execReturn.Stdout != "" { if containerstatus[0] == "Created" { - log.G(h.Ctx).Info("-- Container " + container.Name + " is going ready...") - resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) + log.G(h.Ctx).Info("-- Container " + containerName + " is going ready...") + resp[i].Containers = append(resp[i].Containers,v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) } else if containerstatus[0] == "Up" { - log.G(h.Ctx).Info("-- Container " + container.Name + " is running") - resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true}) + log.G(h.Ctx).Info("-- Container " + containerName + " is running") + resp[i].Containers = append(resp[i].Containers,v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true}) } else if containerstatus[0] == "Exited" { - log.G(h.Ctx).Info("-- Container " + container.Name + " has been stopped") - resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false}) + log.G(h.Ctx).Info("-- Container " + containerName + " has been stopped") + resp[i].Containers = append(resp[i].Containers,v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false}) // release all the GPUs from the container - h.GpuManager.Release(container.Name) + h.GpuManager.Release(containerName) } } else { - log.G(h.Ctx).Info("-- Container " + container.Name + " doesn't exist") + log.G(h.Ctx).Info("-- Container " + containerName + " doesn't exist") resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false}) } } diff --git a/pkg/docker/aux.go b/pkg/docker/aux.go index 075fab2..0ac238b 100644 --- a/pkg/docker/aux.go +++ b/pkg/docker/aux.go @@ -31,23 +31,27 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data [] mountedData := "" for _, podData := range data { - err := os.MkdirAll(config.DataRootFolder+podData.Pod.Namespace+"-"+string(podData.Pod.UID), os.ModePerm) + + podUID := string(podData.Pod.UID) + podNamespace := string(podData.Pod.UID) + + err := os.MkdirAll(config.DataRootFolder+podData.Pod.Namespace+"-"+podUID, os.ModePerm) if err != nil { log.G(Ctx).Error(err) return "", err } else { - log.G(Ctx).Info("-- Created directory " + config.DataRootFolder + podData.Pod.Namespace + "-" + string(podData.Pod.UID)) + log.G(Ctx).Info("-- Created directory " + config.DataRootFolder + podData.Pod.Namespace + "-" + podUID) } - log.G(Ctx).Info("pod data values: " + fmt.Sprintf("%+v", podData)) - for _, cont := range podData.Containers { + containerName := podNamespace + "-" + podUID + "-" + container.Name + log.G(Ctx).Info("cont values: " + fmt.Sprintf("%+v", cont)) log.G(Ctx).Info("-- Inside Preparing mountpoints for " + cont.Name) for _, cfgMap := range cont.ConfigMaps { - if container.Name == cont.Name { + if containerName == podNamespace + "-" + podUID + "-" + cont.Name { log.G(Ctx).Info("-- Mounting ConfigMap " + cfgMap.Name) paths, err := mountData(Ctx, config, podData.Pod, cfgMap, container) if err != nil { @@ -61,7 +65,7 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data [] } for _, secret := range cont.Secrets { - if container.Name == cont.Name { + if containerName == podNamespace + "-" + podUID + "-" + cont.Name { paths, err := mountData(Ctx, config, podData.Pod, secret, container) if err != nil { log.G(Ctx).Error("Error mounting Secret " + secret.Name) @@ -74,7 +78,7 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data [] } for _, emptyDir := range cont.EmptyDirs { - if container.Name == cont.Name { + if containerName == podNamespace + "-" + podUID + "-" + cont.Name { paths, err := mountData(Ctx, config, podData.Pod, emptyDir, container) if err != nil { log.G(Ctx).Error("Error mounting EmptyDir " + emptyDir)