Skip to content

Commit

Permalink
#1 - first commit and try to fix the container name problem
Browse files Browse the repository at this point in the history
  • Loading branch information
Bianco95 committed Apr 22, 2024
1 parent c79b3f4 commit 55db431
Show file tree
Hide file tree
Showing 5 changed files with 62 additions and 37 deletions.
31 changes: 18 additions & 13 deletions pkg/docker/Create.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) {

for _, data := range req {

podUID := string(data.Pod.UID)
podNamespace := string(data.Pod.Namespace)

pathsOfVolumes := make(map[string]string)

for _, volume := range data.Pod.Spec.Volumes {
Expand All @@ -61,6 +64,8 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) {

for _, container := range data.Pod.Spec.Containers {

containerName := podNamespace + "-" + podUID + "-" + container.Name

var isGpuRequested bool = false
var additionalGpuArgs []string

Expand All @@ -72,7 +77,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) {

isGpuRequested = true

log.G(h.Ctx).Info("Container " + container.Name + " is requesting " + val.String() + " GPU")
log.G(h.Ctx).Info("Container " + containerName + " is requesting " + val.String() + " GPU")

numGpusRequestedInt := int(numGpusRequested)
_, err := h.GpuManager.GetAvailableGPUs(numGpusRequestedInt)
Expand All @@ -82,7 +87,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) {
return
}

gpuSpecs, err := h.GpuManager.GetAndAssignAvailableGPUs(numGpusRequestedInt, container.Name)
gpuSpecs, err := h.GpuManager.GetAndAssignAvailableGPUs(numGpusRequestedInt, containerName)
if err != nil {
HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data)
return
Expand All @@ -100,10 +105,10 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) {
additionalGpuArgs = append(additionalGpuArgs, "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES="+gpuUUIDs)

} else {
log.G(h.Ctx).Info("Container " + container.Name + " is not requesting a GPU")
log.G(h.Ctx).Info("Container " + containerName + " is not requesting a GPU")
}

log.G(h.Ctx).Info("- Creating container " + container.Name)
log.G(h.Ctx).Info("- Creating container " + containerName)

var envVars string = ""
// add environment variables to the docker command
Expand Down Expand Up @@ -138,9 +143,9 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) {
}
}

log.G(h.Ctx).Info("- Creating container " + container.Name)
log.G(h.Ctx).Info("- Creating container " + containerName)

cmd := []string{"run", "-d", "--name", container.Name}
cmd := []string{"run", "-d", "--name", containerName}

cmd = append(cmd, envVars)

Expand Down Expand Up @@ -195,34 +200,34 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) {
}

if execReturn.Stdout == "" {
eval := "Conflict. The container name \"/" + container.Name + "\" is already in use"
eval := "Conflict. The container name \"/" + containerName + "\" is already in use"
if strings.Contains(execReturn.Stderr, eval) {
log.G(h.Ctx).Warning("Container named " + container.Name + " already exists. Skipping its creation.")
log.G(h.Ctx).Warning("Container named " + containerName + " already exists. Skipping its creation.")
} else {
log.G(h.Ctx).Error("Unable to create container " + container.Name + " : " + execReturn.Stderr)
log.G(h.Ctx).Error("Unable to create container " + containerName + " : " + execReturn.Stderr)
HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data)
return
}
} else {
log.G(h.Ctx).Info("-- Created container " + container.Name)
log.G(h.Ctx).Info("-- Created container " + containerName)
}

shell = exec.ExecTask{
Command: "docker",
Args: []string{"ps", "-aqf", "name=^" + container.Name + "$"},
Args: []string{"ps", "-aqf", "name=^" + containerName + "$"},
Shell: true,
}

execReturn, err = shell.Execute()
execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "")
if execReturn.Stderr != "" {
log.G(h.Ctx).Error("Failed to retrieve " + container.Name + " ID : " + execReturn.Stderr)
log.G(h.Ctx).Error("Failed to retrieve " + containerName + " ID : " + execReturn.Stderr)
HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data)
return
} else if execReturn.Stdout == "" {
log.G(h.Ctx).Error("Container name not found. Maybe creation failed?")
} else {
log.G(h.Ctx).Debug("-- Retrieved " + container.Name + " ID: " + execReturn.Stdout)
log.G(h.Ctx).Debug("-- Retrieved " + containerName + " ID: " + execReturn.Stdout)
}
}
}
Expand Down
21 changes: 14 additions & 7 deletions pkg/docker/Delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,18 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) {
return
}

podUID := string(pod.PodUID)
podNamespace := string(req.Namespace)


for _, container := range pod.Spec.Containers {
log.G(h.Ctx).Debug("- Deleting container " + container.Name)

containerName := podNamespace + "-" + podUID + "-" + container.Name

log.G(h.Ctx).Debug("- Deleting container " + containerName)

// added a timeout to the stop container command
cmd := []string{"stop", "-t", "10", container.Name}
cmd := []string{"stop", "-t", "10", containerName}
shell := exec.ExecTask{
Command: "docker",
Args: cmd,
Expand All @@ -51,9 +58,9 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) {

if execReturn.Stderr != "" {
if strings.Contains(execReturn.Stderr, "No such container") {
log.G(h.Ctx).Debug("-- Unable to find container " + container.Name + ". Probably already removed? Skipping its removal")
log.G(h.Ctx).Debug("-- Unable to find container " + containerName + ". Probably already removed? Skipping its removal")
} else {
log.G(h.Ctx).Error("-- Error stopping container " + container.Name + ". Skipping its removal")
log.G(h.Ctx).Error("-- Error stopping container " + containerName + ". Skipping its removal")
statusCode = http.StatusInternalServerError
w.WriteHeader(statusCode)
w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs"))
Expand All @@ -73,18 +80,18 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) {
execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "")

if execReturn.Stderr != "" {
log.G(h.Ctx).Error("-- Error deleting container " + container.Name)
log.G(h.Ctx).Error("-- Error deleting container " + containerName)
statusCode = http.StatusInternalServerError
w.WriteHeader(statusCode)
w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs"))
return
} else {
log.G(h.Ctx).Info("- Deleted container " + container.Name)
log.G(h.Ctx).Info("- Deleted container " + containerName)
}
}

// check if the container has GPU devices attacched using the GpuManager and release them
h.GpuManager.Release(container.Name)
h.GpuManager.Release(containerName)

os.RemoveAll(h.Config.DataRootFolder + pod.Namespace + "-" + string(pod.UID))
}
Expand Down
9 changes: 6 additions & 3 deletions pkg/docker/GetLogs.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,16 @@ func (h *SidecarHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request)
return
}

//req = test
podUID := string(req.PodUID)
podNamespace := string(req.Namespace)

containerName := podNamespace + "-" + podUID + "-" + req.ContainerName

var cmd *OSexec.Cmd
if req.Opts.Timestamps {
cmd = OSexec.Command("docker", "logs", "-t", req.ContainerName)
cmd = OSexec.Command("docker", "logs", "-t", containerName)
} else {
cmd = OSexec.Command("docker", "logs", req.ContainerName)
cmd = OSexec.Command("docker", "logs", containerName)
}

output, err := cmd.CombinedOutput()
Expand Down
30 changes: 18 additions & 12 deletions pkg/docker/Status.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,16 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) {
}

for i, pod := range req {
resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodUID: string(pod.UID), PodNamespace: pod.Namespace})

podUID := string(pod.UID)
podNamespace := string(pod.Namespace)

resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodUID: podUID, PodNamespace: podNamespace})
for _, container := range pod.Spec.Containers {
log.G(h.Ctx).Debug("- Getting status for container " + container.Name)
cmd := []string{"ps -af name=^" + container.Name + "$ --format \"{{.Status}}\""}
containerName := podNamespace + "-" + podUID + "-" + container.Name

log.G(h.Ctx).Debug("- Getting status for container " + containerName)
cmd := []string{"ps -af name=^" + containerName + "$ --format \"{{.Status}}\""}

shell := exec.ExecTask{
Command: "docker",
Expand All @@ -63,20 +69,20 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) {
// TODO: why first container?
if execReturn.Stdout != "" {
if containerstatus[0] == "Created" {
log.G(h.Ctx).Info("-- Container " + container.Name + " is going ready...")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false})
log.G(h.Ctx).Info("-- Container " + containerName + " is going ready...")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: containerName, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false})
} else if containerstatus[0] == "Up" {
log.G(h.Ctx).Info("-- Container " + container.Name + " is running")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true})
log.G(h.Ctx).Info("-- Container " + containerName + " is running")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: containerName, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true})
} else if containerstatus[0] == "Exited" {
log.G(h.Ctx).Info("-- Container " + container.Name + " has been stopped")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false})
log.G(h.Ctx).Info("-- Container " + containerName + " has been stopped")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: containerName, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false})
// release all the GPUs from the container
h.GpuManager.Release(container.Name)
h.GpuManager.Release(containerName)
}
} else {
log.G(h.Ctx).Info("-- Container " + container.Name + " doesn't exist")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false})
log.G(h.Ctx).Info("-- Container " + containerName + " doesn't exist")
resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: containerName, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false})
}
}
}
Expand Down
8 changes: 6 additions & 2 deletions pkg/docker/aux.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,16 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data []
mountedData := ""

for _, podData := range data {
err := os.MkdirAll(config.DataRootFolder+podData.Pod.Namespace+"-"+string(podData.Pod.UID), os.ModePerm)

podUID := string(podData.Pod.UID)
podNamespace := string(podData.Pod.Namespace)

err := os.MkdirAll(config.DataRootFolder+podData.Pod.Namespace+"-"+podUID, os.ModePerm)
if err != nil {
log.G(Ctx).Error(err)
return "", err
} else {
log.G(Ctx).Info("-- Created directory " + config.DataRootFolder + podData.Pod.Namespace + "-" + string(podData.Pod.UID))
log.G(Ctx).Info("-- Created directory " + config.DataRootFolder + podData.Pod.Namespace + "-" + podUID)
}

log.G(Ctx).Info("pod data values: " + fmt.Sprintf("%+v", podData))
Expand Down

0 comments on commit 55db431

Please sign in to comment.