Unable to rollout NetBox after upgrade from 5.0.0-beta.160

[Sharvesh0306]

The Helm chart version

Netbox-5.0.18

Environment Versions

Argo CD: v2.8.0+804d4b8
❯ kubectl version
Client Version: v1.30.2
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
Server Version: v1.28.3+rke2r2

Custom chart values

global:
extraDeploy: []
image:
  registry: ghcr.io
  repository: netbox-community/netbox
  pullPolicy: IfNotPresent
  tag: ""
  digest: ""
  pullSecrets: []
superuser:
  name: admin
  email: admin@example.com
  password: ""
  apiToken: ""
  existingSecret: "netbox-secrets"
allowedHosts:
  - "*"
allowedHostsIncludesPodIP: true
admins: []
allowTokenRetrieval: false
authPasswordValidators: []
allowedUrlSchemes:
  - file
  - ftp
  - ftps
  - http
  - https
  - irc
  - mailto
  - sftp
  - ssh
  - tel
  - telnet
  - tftp
  - vnc
  - xmpp
banner:
  top: ""
  bottom: ""
  login: ""
basePath: ""
changelogRetention: 90
customValidators: {}
defaultUserPreferences: {}
cors:
  originAllowAll: false
  originWhitelist: []
  originRegexWhitelist: []

csrf:
  cookieName: csrftoken
  trustedOrigins: []
defaultLanguage: en-us
dataUploadMaxMemorySize: 2621440
debug: true
dbWaitDebug: false
email:
  server: localhost
  port: 25
  username: ""
  password: ""
  useSSL: false
  useTLS: false
  sslCertFile: ""
  sslKeyFile: ""

  timeout: 10
  from: ""
  existingSecretName: ""
  existingSecretKey: email-password
enforceGlobalUnique: false
exemptViewPermissions: []
fieldChoices: {}
fileUploadMaxMemorySize: 2621440
graphQlEnabled: true
httpProxies: {}
internalIPs: ["127.0.0.1", "::1"]
jobRetention: 90
logging: {}
loginPersistence: false
loginRequired: false
loginTimeout: 1209600
logoutRedirectUrl: home
maintenanceMode: false
mapsUrl: "https://maps.google.com/?q="
maxPageSize: 1000
storageBackend: ""
storageConfig: {}
paginateCount: 50
plugins: []
pluginsConfig: {}
powerFeedDefaultAmperage: 15
powerFeedMaxUtilisation: 80
powerFeedDefaultVoltage: 120
preferIPv4: false
rackElevationDefaultUnitHeight: 22
rackElevationDefaultUnitWidth: 220
remoteAuth:
  enabled: true
  backends:

    - social_core.backends.okta_openidconnect.OktaOpenIdConnect
  header: HTTP_REMOTE_USER
  autoCreateUser: true
  autoCreateGroups: true
  defaultGroups: []
  defaultPermissions: {}
  groupSyncEnabled: true
  groupHeader: HTTP_REMOTE_USER_GROUP


  staffGroups: []
  staffUsers: []
  groupSeparator: "|"
releaseCheck:
  url: "https://api.github.com/repos/netbox-community/netbox/releases"
rqDefaultTimeout: 300
sessionCookieName: sessionid
enableLocalization: false
timeZone: UTC
dateFormat: "N j, Y"
shortDateFormat: "Y-m-d"
timeFormat: "g:i a"
shortTimeFormat: "H:i:s"
dateTimeFormat: "N j, Y g:i a"
shortDateTimeFormat: "Y-m-d H:i"
extraConfig:
  - values:
      SOCIAL_AUTH_PIPELINE: [
        "social_core.pipeline.social_auth.social_details",
        "social_core.pipeline.social_auth.social_uid",
        "social_core.pipeline.social_auth.auth_allowed",
        "social_core.pipeline.social_auth.social_user",
        "social_core.pipeline.social_auth.associate_by_email",
        "social_core.pipeline.user.create_user",
        "social_core.pipeline.social_auth.associate_user",
        "social_core.pipeline.social_auth.load_extra_data",
        "social_core.pipeline.user.user_details",
        "netbox.sso_pipeline_roles.map_groups"
      ]
  - secret:
      secretName: netbox-secrets
      items:
      - key: okta-secrets-file
        path: extra-1.yaml
secretKey: ""
existingSecret: ""
overrideUnitConfig: {}
command: []
args: []
replicaCount: 1
persistence:
  enabled: true
  storageClass: "vast"
  subPath: ""
  accessMode: ReadWriteOnce
  size: 1Gi
  existingClaim: ""
  annotations: {}
reportsPersistence:
  enabled: false
  existingClaim: ""
  subPath: ""
  storageClass: "vast"
  accessMode: ReadWriteOnce
  size: 1Gi
  annotations: {}
scriptsPersistence:
  enabled: false
  existingClaim: ""
  subPath: ""
  storageClass: ""
  accessMode: ReadWriteOnce
  size: 1Gi
  annotations: {}
updateStrategy:
  type: RollingUpdate
serviceAccount:
  create: true
  annotations: {}
  name: ""
  automountServiceAccountToken: false
hostAliases: []
extraVolumes:
  - name: sso-pipeline-roles
    configMap:
      name: sso-pipeline-roles
extraVolumeMounts:
  - name: sso-pipeline-roles
    mountPath: /opt/netbox/netbox/netbox/sso_pipeline_roles.py
    subPath: sso_pipeline_roles.py
    readOnly: true
sidecars: []
initContainers: []
podLabels: {}
podAnnotations: {}
affinity: {}
nodeSelector: {}
tolerations: []
priorityClassName: ""
schedulerName: ""
terminationGracePeriodSeconds:
topologySpreadConstraints: []
resourcesPreset: "large"
resources: {}
podSecurityContext:
  enabled: true
  fsGroupChangePolicy: Always
  sysctls: []
  supplementalGroups: []
  fsGroup: 1000
securityContext:
  enabled: true
  seLinuxOptions: {}
  runAsUser: 1000
  runAsGroup: 1000
  runAsNonRoot: true
  privileged: false
  readOnlyRootFilesystem: true
  allowPrivilegeEscalation: false
  capabilities:
    drop: ["ALL"]
  seccompProfile:
    type: "RuntimeDefault"
automountServiceAccountToken: false
livenessProbe:
  enabled: true
  initialDelaySeconds: 0
  periodSeconds: 10
  timeoutSeconds: 1
  failureThreshold: 3
  successThreshold: 1
readinessProbe:
  enabled: true
  initialDelaySeconds: 0
  periodSeconds: 10
  timeoutSeconds: 1
  failureThreshold: 3
  successThreshold: 1
startupProbe:
  enabled: true
  initialDelaySeconds: 120
  periodSeconds: 10
  timeoutSeconds: 1
  failureThreshold: 10
  successThreshold: 1
customLivenessProbe: {}
customReadinessProbe: {}
customStartupProbe: {}
lifecycleHooks: {}
revisionHistoryLimit: 4
service:
  annotations: {}
  type: ClusterIP
  port: 80
  nodePort: ""
  clusterIP: ""
  externalTrafficPolicy: Cluster
  loadBalancerIP: ""
  loadBalancerSourceRanges: []
  loadBalancerClass: ""
  externalIPs: []
  clusterIPs: []
  ipFamilyPolicy: ""
  sessionAffinity: None
  sessionAffinityConfig: {}
ingress:
  enabled: true
  pathType: ImplementationSpecific
  className: "nginx"
  annotations:
    kubernetes.io/tls-acme: "true"
    cert-manager.io/common-name: netbox.test.io
    cert-manager.io/usages: server auth, client auth
    external-dns.alpha.kubernetes.io/ingress-hostname-source: annotation-only
    external-dns.alpha.kubernetes.io/hostname: netbox.test.io
    nginx.ingress.kubernetes.io/server-alias:
  hosts:
    - host: netbox.test.io
      paths:
        - /
  tls:
    - secretName: netbox-tls
      hosts:
        - netbox.test.io
metrics:
  enabled: false
  serviceMonitor:
    enabled: false
    honorLabels: false
    interval: ""
    scrapeTimeout: ""
    metricRelabelings: []
    relabelings: []
    selector: {}
    additionalLabels: {}
postgresql:
  enabled: false
  auth:
    username: netbox
    database: netbox
    existingSecret: ""
externalDatabase:
  host: netbox.postgres.test.io
  port: 9600
  database: netbox
  username: "netbox"
  password: ""
  existingSecretName: netbox-secrets
  existingSecretKey: postgres-password

  sslMode: prefer
  connMaxAge: 300
  disableServerSideCursors: false
  targetSessionAttrs: read-write
redis:
  enabled: true
tasksRedis:
  database: 0
  ssl: false
  insecureSkipTlsVerify: false

  caCertPath: ""


  host: netbox-redis
  port: 6379
  sentinels: []

  sentinelService: netbox-redis
  sentinelTimeout: 300
  username: ""
  password: ""
  existingSecretName: "netbox-secrets"
  existingSecretKey: redis-password
cachingRedis:
  database: 1
  ssl: false
  insecureSkipTlsVerify: false

  caCertPath: ""


  host: netbox-redis
  port: 6379
  sentinels: []

  sentinelService: netbox-redis
  sentinelTimeout: 300
  username: ""
  password: ""
  existingSecretName: "netbox-secrets"
  existingSecretKey: redis-password
autoscaling:
  enabled: false
  minReplicas: 1
  maxReplicas: 100
  targetCPUUtilizationPercentage: 80

init:
  image:
    registry: docker.io
    repository: busybox
    tag: 1.37.0
    digest: ""
    pullPolicy: IfNotPresent
    pullSecrets: []
  resourcesPreset: "nano"
  resources: {}
  securityContext:
    enabled: true
    seLinuxOptions: {}
    seccompProfile:
      type: RuntimeDefault
    capabilities:
      drop:
        - ALL
    readOnlyRootFilesystem: true
    allowPrivilegeEscalation: false
    runAsNonRoot: true
    runAsUser: 1000
    runAsGroup: 1000
test:
  image:
    registry: docker.io
    repository: busybox
    tag: 1.37.0
    digest: ""
    pullPolicy: IfNotPresent
    pullSecrets: []
  resourcesPreset: "nano"
  resources: {}
  securityContext:
    enabled: false
    seLinuxOptions: {}
    seccompProfile:
      type: RuntimeDefault
    capabilities:
      drop:
        - ALL
    readOnlyRootFilesystem: true
    allowPrivilegeEscalation: false
    runAsNonRoot: true
    runAsUser: 1000
    runAsGroup: 1000
housekeeping:
  enabled: true
  schedule: "0 0 * * *"
  successfulJobsHistoryLimit: 5
  failedJobsHistoryLimit: 5
  command:
    - /opt/netbox/venv/bin/python
    - /opt/netbox/netbox/manage.py
    - housekeeping
  args: []
  podAnnotations: {}
  podSecurityContext:
    enabled: true
    fsGroup: 1000
    fsGroupChangePolicy: Always
    sysctls: []
    supplementalGroups: []
  securityContext:
    enabled: true
    seLinuxOptions: {}
    seccompProfile:
      type: "RuntimeDefault"
    capabilities:
      drop:
        - ALL
    privileged: false
    readOnlyRootFilesystem: true
    allowPrivilegeEscalation: false
    runAsNonRoot: true
    runAsUser: 1000
    runAsGroup: 1000
  resourcesPreset: "none"
  resources: {}
  readOnlyPersistence: false
  extraEnvs: []
  extraVolumes: []
  extraVolumeMounts: []
  sidecars: []
  initContainers: []
  affinity: {}
  nodeSelector: {}
  tolerations: []
  podLabels: {}
  automountServiceAccountToken: false
  concurrencyPolicy: Forbid
  restartPolicy: OnFailure
  suspend: false
worker:
  enabled: false
  command:
    - /opt/netbox/venv/bin/python
    - /opt/netbox/netbox/manage.py
    - rqworker
  args: []
  replicaCount: 1
  podLabels: {}
  podAnnotations: {}
  podSecurityContext:
    enabled: true
    fsGroup: 1000
    fsGroupChangePolicy: Always
    sysctls: []
    supplementalGroups: []
  securityContext:
    enabled: true
    seLinuxOptions: {}
    seccompProfile:
      type: "RuntimeDefault"
    capabilities:
      drop:
        - ALL
    privileged: false
    readOnlyRootFilesystem: true
    allowPrivilegeEscalation: false
    runAsNonRoot: true
    runAsUser: 1000
    runAsGroup: 1000
  resourcesPreset: "none"
  resources: {}
  readOnlyPersistence: false
  automountServiceAccountToken: false
  affinity: {}
  nodeSelector: {}
  tolerations: []
  priorityClassName: ""
  schedulerName: ""
  terminationGracePeriodSeconds:
  topologySpreadConstraints: []
  hostAliases: []
  updateStrategy:
    type: RollingUpdate
  autoscaling:
    enabled: false
    minReplicas: 1
    maxReplicas: 100
    targetCPUUtilizationPercentage: 80

  extraEnvs: []
  extraVolumes: []
  extraVolumeMounts: []
  sidecars: []
  initContainers: []

Current Behavior & Steps to Reproduce

The version upgrade is not rolling out the new Netbox Pods and its crash looping with an error : redis.exceptions.ResponseError: unknown command 'FLUSHDB'

Expected Behavior

Successfully rollout NetBox Helm pod

NetBox Logs

redis.exceptions.ResponseError: unknown command 'FLUSHDB', with args beginning with:[ Use DB_WAIT_DEBUG=1 in netbox.env to print full traceback for errors here ]⏳ Waiting on DB... (0s / 30s)redis.exceptions.ResponseError: unknown command 'FLUSHDB', with args beginning with:[ Use DB_WAIT_DEBUG=1 in netbox.env to print full traceback for errors here ]⏳ Waiting on DB... (3s / 30s)redis.exceptions.ResponseError: unknown command 'FLUSHDB', with args beginning with:[ Use DB_WAIT_DEBUG=1 in netbox.env to print full traceback for errors here ]⏳ Waiting on DB... (6s / 30s)redis.exceptions.ResponseError: unknown command 'FLUSHDB', with args beginning with:[ Use DB_WAIT_DEBUG=1 in netbox.env to print full traceback for errors here ]⏳ Waiting on DB... (9s / 30s)

[LeoColomb]

Thanks for filing this issue, @Sharvesh0306.
Can you try inserting these values:

redis:
  enabled: true
  master:
    disableCommands: []
...

[Sharvesh0306]

Thanks for filing this issue, @Sharvesh0306. Can you try inserting these values:

redis:
enabled: true
master:
disableCommands: []
...

Thanks @LeoColomb. This seem to get me past the redis error, now I encountered a new error:

Static Media Failure
The following static media file failed to load: [netbox.css](https://nonprod.netbox.test.io/static/netbox.css)
Check the following:

manage.py collectstatic was run during the most recent upgrade. This installs the most recent iteration of each static file into the static root path.
The HTTP service (e.g. nginx or Apache) is configured to serve files from the STATIC_ROOT path. Refer to [the installation documentation](https://docs.netbox.dev/en/stable/installation/) for further guidance.
STATIC_URL: /static/
The file netbox.css exists in the static root directory and is readable by the HTTP server.
Click [here](https://nonprod.netbox.test.io/) to attempt loading NetBox again.

Pod Logs:

Session data corrupted127.0.0.1 - - [22/Jan/2025:16:38:32 +0000] "GET /media-failure/?filename=netbox.css HTTP/1.1" 200 15157 "https://nonprod.netbox.test.io/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
2025/01/22 16:38:32 [alert] 7#7 process 285 exited on signal 11 (core dumped)
2025/01/22 16:38:32 [alert] 7#7 sendmsg(10, -1, -1, 1) failed (32: Broken pipe)
2025/01/22 16:38:32 [alert] 7#7 sendmsg(13, -1, -1, 1) failed (32: Broken pipe)
2025/01/22 16:38:32 [info] 349#349 router started
🧬 loaded config '/etc/netbox/config/configuration.py'
🧬 loaded config '/etc/netbox/config/extra.py'
🧬 loaded config '/etc/netbox/config/logging.py'
🧬 loaded config '/etc/netbox/config/plugins.py'
🧬 loaded config '/etc/netbox/config/configuration.py'
🧬 loaded config '/etc/netbox/config/extra.py'
🧬 loaded config '/etc/netbox/config/logging.py'
🧬 loaded config '/etc/netbox/config/plugins.py'
2025/01/22 16:38:32 [info] 349#349 OpenSSL 3.0.13 30 Jan 2024, 300000d0
2025/01/22 16:38:32 [info] 350#350 "netbox" prototype started
2025/01/22 16:38:32 [info] 351#351 "netbox" application started
2025/01/22 16:38:34 [notice] 286#286 app process 326 exited with code 0
2025/01/22 16:38:34 [notice] 286#286 app process 287 exited with code 0
2025/01/22 16:38:34 [notice] 7#7 process 286 exited with code 0
2025/01/22 16:38:35 [alert] 7#7 process 349 exited on signal 11 (core dumped)2025/01/22 16:38:35 [alert] 7#7 sendmsg(11, -1, -1, 1) failed (32: Broken pipe)
2025/01/22 16:38:35 [alert] 218#218 failed to apply previous configuration
🧬 loaded config '/etc/netbox/config/configuration.py'
🧬 loaded config '/etc/netbox/config/extra.py'
🧬 loaded config '/etc/netbox/config/logging.py'
🧬 loaded config '/etc/netbox/config/plugins.py'
2025/01/22 16:38:35 [info] 390#390 router started
2025/01/22 16:38:35 [info] 390#390 OpenSSL 3.0.13 30 Jan 2024, 300000d0
2025/01/22 16:38:35 [notice] 350#350 app process 351 exited with code 0
2025/01/22 16:38:35 [notice] 7#7 process 350 exited with code 0
2025/01/22 16:38:55 [notice] 7#7 process 218 exited with code 0
2025/01/22 16:38:55 [notice] 7#7 process 390 exited with code 0

[LeoColomb]

@Sharvesh0306 Are you able to connect through the web UI?
Can you provide the pod spec, events included?