Skip to content

Commit

Permalink
minor fix on gpu assignment logic and mount of configmap (path was no…
Browse files Browse the repository at this point in the history
…t correct)
  • Loading branch information
Bianco95 committed Apr 24, 2024
1 parent d6df203 commit 6a04059
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 25 deletions.
54 changes: 33 additions & 21 deletions pkg/docker/Create.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,39 +75,48 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) {

log.G(h.Ctx).Infof("Number of GPU requested: %d", numGpusRequested)

isGpuRequested = true
// if the container is requesting 0 GPU, skip the GPU assignment
if numGpusRequested == 0 {
log.G(h.Ctx).Info("Container " + containerName + " is not requesting a GPU")

log.G(h.Ctx).Info("Container " + containerName + " is requesting " + val.String() + " GPU")
} else {

numGpusRequestedInt := int(numGpusRequested)
_, err := h.GpuManager.GetAvailableGPUs(numGpusRequestedInt)
log.G(h.Ctx).Info("Container " + containerName + " is requesting " + val.String() + " GPU")

if err != nil {
HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data)
return
}
isGpuRequested = true

gpuSpecs, err := h.GpuManager.GetAndAssignAvailableGPUs(numGpusRequestedInt, containerName)
if err != nil {
HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data)
return
}
numGpusRequestedInt := int(numGpusRequested)
_, err := h.GpuManager.GetAvailableGPUs(numGpusRequestedInt)

var gpuUUIDs string = ""
for _, gpuSpec := range gpuSpecs {
if gpuSpec.UUID == gpuSpecs[len(gpuSpecs)-1].UUID {
gpuUUIDs += strconv.Itoa(gpuSpec.Index)
} else {
gpuUUIDs += strconv.Itoa(gpuSpec.Index) + ","
if err != nil {
HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data)
return
}

gpuSpecs, err := h.GpuManager.GetAndAssignAvailableGPUs(numGpusRequestedInt, containerName)
if err != nil {
HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data)
return
}

var gpuUUIDs string = ""
for _, gpuSpec := range gpuSpecs {
if gpuSpec.UUID == gpuSpecs[len(gpuSpecs)-1].UUID {
gpuUUIDs += strconv.Itoa(gpuSpec.Index)
} else {
gpuUUIDs += strconv.Itoa(gpuSpec.Index) + ","
}
}
}

additionalGpuArgs = append(additionalGpuArgs, "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES="+gpuUUIDs)
additionalGpuArgs = append(additionalGpuArgs, "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES="+gpuUUIDs)
}

} else {
log.G(h.Ctx).Info("Container " + containerName + " is not requesting a GPU")
}

log.G(h.Ctx).Info("-- Preparing environment variables for " + containerName)

var envVars string = ""
// add environment variables to the docker command
for _, envVar := range container.Env {
Expand Down Expand Up @@ -166,6 +175,9 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) {
HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data)
return
}
// print the mounts for debugging
log.G(h.Ctx).Info("Mounts: " + mounts)

cmd = append(cmd, mounts)
}

Expand Down
12 changes: 8 additions & 4 deletions pkg/docker/aux.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,10 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data []

log.G(Ctx).Info("-- Inside Preparing mountpoints for " + cont.Name)
for _, cfgMap := range cont.ConfigMaps {
if containerName == podNamespace + "-" + podUID + "-" + cont.Name {
if containerName == podNamespace+"-"+podUID+"-"+cont.Name {
log.G(Ctx).Info("-- Mounting ConfigMap " + cfgMap.Name)
paths, err := mountData(Ctx, config, podData.Pod, cfgMap, container)
log.G(Ctx).Info("-- Paths: " + strings.Join(paths, ","))
if err != nil {
log.G(Ctx).Error("Error mounting ConfigMap " + cfgMap.Name)
return "", errors.New("Error mounting ConfigMap " + cfgMap.Name)
Expand All @@ -65,7 +66,7 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data []
}

for _, secret := range cont.Secrets {
if containerName == podNamespace + "-" + podUID + "-" + cont.Name {
if containerName == podNamespace+"-"+podUID+"-"+cont.Name {
paths, err := mountData(Ctx, config, podData.Pod, secret, container)
if err != nil {
log.G(Ctx).Error("Error mounting Secret " + secret.Name)
Expand All @@ -78,7 +79,7 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data []
}

for _, emptyDir := range cont.EmptyDirs {
if containerName == podNamespace + "-" + podUID + "-" + cont.Name {
if containerName == podNamespace+"-"+podUID+"-"+cont.Name {
paths, err := mountData(Ctx, config, podData.Pod, emptyDir, container)
if err != nil {
log.G(Ctx).Error("Error mounting EmptyDir " + emptyDir)
Expand Down Expand Up @@ -141,7 +142,10 @@ func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod,

if mount.Data != nil {
for key := range mount.Data {
path := filepath.Join(wd+podConfigMapDir, key)

log.G(Ctx).Info("Key: " + key)

path := filepath.Join(podConfigMapDir, key)
path += (":" + mountSpec.MountPath + "/" + key + " ")
configMapNamePaths = append(configMapNamePaths, path)
}
Expand Down

0 comments on commit 6a04059

Please sign in to comment.