feat: nautobot JobQueue setup gitops way

.yamllint.yaml

@@ -4,6 +4,7 @@ ignore:
   - components/etcdbackup/templates/
   - charts/argocd-understack/templates/
   - charts/nautobot-api-tokens/templates/
+  - charts/nautobot-job-queues/templates/
   - charts/site-workflows/templates/
   - charts/undersync/templates/
 

charts/argocd-understack/templates/application-nautobot-job-queues.yaml

@@ -0,0 +1,37 @@
+{{- if eq (include "understack.isEnabled" (list $.Values.global "nautobot_job_queues")) "true" }}
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ printf "%s-%s" $.Release.Name "nautobot-job-queues" }}
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/compare-options: ServerSideDiff=true,IncludeMutationWebhook=true
+{{- include "understack.appLabelsBlock" $ | nindent 2 }}
+spec:
+  destination:
+    namespace: nautobot
+    server: {{ $.Values.cluster_server }}
+  project: understack
+  sources:
+  - repoURL: {{ include "understack.understack_url" $ }}
+    targetRevision: {{ include "understack.understack_ref" $ }}
+    path: charts/nautobot-job-queues
+    helm:
+      ignoreMissingValueFiles: true
+      valueFiles:
+      - $deploy/{{ include "understack.deploy_path" $ }}/nautobot-job-queues/values.yaml
+  - path: {{ include "understack.deploy_path" $ }}/nautobot-job-queues
+    ref: deploy
+    repoURL: {{ include "understack.deploy_url" $ }}
+    targetRevision: {{ include "understack.deploy_ref" $ }}
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - ServerSideApply=true
+    - RespectIgnoreDifferences=true
+    - ApplyOutOfSyncOnly=true
+{{- end }}

charts/argocd-understack/values.yaml

@@ -154,6 +154,12 @@ global:
     # @default -- false
     enabled: false
 
+  # -- Nautobot Celery JobQueue bootstrap jobs
+  nautobot_job_queues:
+    # -- Enable/disable deploying
+    # @default -- false
+    enabled: false
+
   # -- Nautobot Operator for Kubernetes
   nautobotop:
     # -- Enable/disable deploying Nautobot Operator

charts/nautobot-job-queues/Chart.yaml

@@ -0,0 +1,12 @@
+apiVersion: v2
+name: nautobot-job-queues
+description: Ensure Nautobot Celery JobQueue records from Helm values
+
+type: application
+
+version: 0.1.0
+# renovate: datasource=docker depName=networktocode/nautobot versioning=semver
+appVersion: "3.0.7"
+
+maintainers:
+  - name: rackerlabs

charts/nautobot-job-queues/ci/example.yaml

@@ -0,0 +1,2 @@
+queues:
+  - site-dc

charts/nautobot-job-queues/templates/_helpers.tpl

@@ -0,0 +1,49 @@
+{{/* Expand the name of the chart. */}}
+{{- define "nautobot-job-queues.name" -}}
+{{- .Chart.Name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/* Create a default fully qualified app name. */}}
+{{- define "nautobot-job-queues.fullname" -}}
+{{- if contains .Chart.Name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+
+{{/* Chart label value. */}}
+{{- define "nautobot-job-queues.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/* Common labels. */}}
+{{- define "nautobot-job-queues.labels" -}}
+helm.sh/chart: {{ include "nautobot-job-queues.chart" . }}
+{{ include "nautobot-job-queues.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/* Selector labels. */}}
+{{- define "nautobot-job-queues.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "nautobot-job-queues.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/* Script config map name. */}}
+{{- define "nautobot-job-queues.scriptConfigMapName" -}}
+{{- printf "%s-script" (include "nautobot-job-queues.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/* Desired JobQueue config map name. */}}
+{{- define "nautobot-job-queues.desiredConfigMapName" -}}
+{{- printf "%s-desired" (include "nautobot-job-queues.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/* Ensure job name. */}}
+{{- define "nautobot-job-queues.jobName" -}}
+{{- printf "%s-ensure" (include "nautobot-job-queues.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end }}

charts/nautobot-job-queues/templates/configmap-desired-queues.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "nautobot-job-queues.desiredConfigMapName" . }}
+  labels:
+    {{- include "nautobot-job-queues.labels" . | nindent 4 }}
+data:
+  job-queues.json: |
+    {{ .Values.queues | toJson }}

charts/nautobot-job-queues/templates/configmap-script.yaml

@@ -0,0 +1,58 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "nautobot-job-queues.scriptConfigMapName" . }}
+  labels:
+    {{- include "nautobot-job-queues.labels" . | nindent 4 }}
+data:
+  ensure.py: |
+    import json
+
+    from django.apps import apps
+
+
+    def _load_desired(path):
+      with open(path, "r", encoding="utf-8") as fp:
+        raw = json.load(fp)
+      if not isinstance(raw, list):
+        raise RuntimeError(f"expected a JSON list in {path}")
+      for index, name in enumerate(raw):
+        if not isinstance(name, str) or not name:
+          raise RuntimeError(f"queue entry {index} must be a non-empty string")
+      return raw
+
+
+    JobQueue = apps.get_model("extras", "JobQueue")
+
+    queue_type = "celery"
+    created = 0
+    updated = 0
+    unchanged = 0
+
+    for name in _load_desired("/desired/job-queues.json"):
+      job_queue = JobQueue.objects.filter(name=name).first()
+
+      if job_queue is None:
+        job_queue = JobQueue.objects.create(name=name, queue_type=queue_type)
+        created += 1
+        print(f"Created Nautobot JobQueue '{job_queue.name}' ({job_queue.queue_type})")
+        continue
+
+      if job_queue.queue_type != queue_type:
+        old_queue_type = job_queue.queue_type
+        job_queue.queue_type = queue_type
+        job_queue.save(update_fields=["queue_type"])
+        updated += 1
+        print(
+          f"Updated Nautobot JobQueue '{job_queue.name}' queue_type "
+          f"from '{old_queue_type}' to '{job_queue.queue_type}'"
+        )
+        continue
+
+      unchanged += 1
+      print(f"Nautobot JobQueue '{job_queue.name}' already exists")
+
+    print(
+      "Nautobot JobQueue ensure complete: "
+      f"{created} created, {updated} updated, {unchanged} unchanged"
+    )

charts/nautobot-job-queues/templates/job.yaml

@@ -0,0 +1,57 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "nautobot-job-queues.jobName" . }}
+  labels:
+    {{- include "nautobot-job-queues.labels" . | nindent 4 }}
+  annotations:
+    argocd.argoproj.io/hook: "PostSync"
+    argocd.argoproj.io/hook-delete-policy: "BeforeHookCreation"
+    argocd.argoproj.io/sync-wave: "10"
+spec:
+  backoffLimit: 3
+  ttlSecondsAfterFinished: 300
+  template:
+    metadata:
+      labels:
+        {{- include "nautobot-job-queues.selectorLabels" . | nindent 8 }}
+        nautobot-job-queues/job: "ensure"
+    spec:
+      serviceAccountName: nautobot
+      restartPolicy: OnFailure
+      containers:
+        - name: ensure
+          image: {{ printf "ghcr.io/nautobot/nautobot:%s" .Chart.AppVersion | quote }}
+          imagePullPolicy: IfNotPresent
+          command:
+            - /bin/bash
+            - -lc
+          args:
+            - nautobot-server shell --interface python < /scripts/ensure.py
+          env:
+            - name: NAUTOBOT_DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.dbPasswordSecretRef.name | quote }}
+                  key: {{ .Values.dbPasswordSecretRef.key | quote }}
+          envFrom:
+            - configMapRef:
+                name: nautobot-env
+            - secretRef:
+                name: nautobot-env
+            - secretRef:
+                name: nautobot-custom-env
+          volumeMounts:
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+            - name: desired
+              mountPath: /desired
+              readOnly: true
+      volumes:
+        - name: scripts
+          configMap:
+            name: {{ include "nautobot-job-queues.scriptConfigMapName" . }}
+        - name: desired
+          configMap:
+            name: {{ include "nautobot-job-queues.desiredConfigMapName" . }}

charts/nautobot-job-queues/values.schema.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "dbPasswordSecretRef": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "key": {
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "required": [
+        "name",
+        "key"
+      ]
+    },
+    "queues": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1
+      }
+    }
+  },
+  "required": [
+    "dbPasswordSecretRef",
+    "queues"
+  ]
+}

charts/nautobot-job-queues/values.yaml

@@ -0,0 +1,8 @@
+# -- Secret reference used to set NAUTOBOT_DB_PASSWORD on the managed job container.
+dbPasswordSecretRef:
+  name: nautobot-env
+  key: NAUTOBOT_DB_PASSWORD
+
+# -- Nautobot JobQueue names to ensure.
+queues: []
+# - site-dc

docs/operator-guide/nautobot-celery-queues.md

@@ -61,54 +61,18 @@ Before any job can be dispatched to a site queue, a `JobQueue` record
 must exist in Nautobot's database. Without it, the API rejects the
 request with a validation error.
 
-### Create via the UI
-
-Navigate to Jobs > Job Queues > Add and create a queue with:
-
-- Name: `site-dc` (must match the worker's `taskQueues` value)
-- Queue Type: `celery`
-
-### Create via the REST API
-
-```bash
-curl -X POST \
-  -H "Authorization: Token $TOKEN" \
-  -H "Content-Type: application/json" \
-  https://nautobot.example.com/api/extras/job-queues/ \
-  --data '{"name": "site-dc", "queue_type": "celery"}'
-```
-
-### Create via pynautobot
-
-```python
-import pynautobot
-
-nb = pynautobot.api("https://nautobot.example.com", token="your-token")
-nb.extras.job_queues.create(name="site-dc", queue_type="celery")
-```
-
-### Automate via Ansible
-
-The `ansible/roles/jobs/tasks/main.yml` role enables jobs but does not
-currently create JobQueues. You can extend it:
-
-{% raw %}
+For understack-managed environments, declare the desired queues in the
+global deploy repo path:
 
 ```yaml
-- name: "Ensure site JobQueue exists"
-  ansible.builtin.uri:
-    url: "{{ nautobot_url }}/api/extras/job-queues/"
-    method: POST
-    headers:
-      Authorization: "Token {{ nautobot_token }}"
-    body_format: json
-    body:
-      name: "{{ site }}"
-      queue_type: "celery"
-    status_code: [200, 201, 400]
+# rax-dev-global/nautobot-job-queues/values.yaml
+queues:
+  - site-dc
 ```
 
-{% endraw %}
+The `nautobot-job-queues` ArgoCD application runs in the global
+cluster and ensures these records exist in the shared Nautobot
+database.
 
 ## Assigning Jobs to Queues