diff --git a/.yamllint.yaml b/.yamllint.yaml index e29c4fbcb..c8ca7a182 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -4,6 +4,7 @@ ignore: - components/etcdbackup/templates/ - charts/argocd-understack/templates/ - charts/nautobot-api-tokens/templates/ + - charts/nautobot-job-queues/templates/ - charts/site-workflows/templates/ - charts/undersync/templates/ diff --git a/charts/argocd-understack/templates/application-nautobot-job-queues.yaml b/charts/argocd-understack/templates/application-nautobot-job-queues.yaml new file mode 100644 index 000000000..a467c80bf --- /dev/null +++ b/charts/argocd-understack/templates/application-nautobot-job-queues.yaml @@ -0,0 +1,37 @@ +{{- if eq (include "understack.isEnabled" (list $.Values.global "nautobot_job_queues")) "true" }} +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ printf "%s-%s" $.Release.Name "nautobot-job-queues" }} + finalizers: + - resources-finalizer.argocd.argoproj.io + annotations: + argocd.argoproj.io/compare-options: ServerSideDiff=true,IncludeMutationWebhook=true +{{- include "understack.appLabelsBlock" $ | nindent 2 }} +spec: + destination: + namespace: nautobot + server: {{ $.Values.cluster_server }} + project: understack + sources: + - repoURL: {{ include "understack.understack_url" $ }} + targetRevision: {{ include "understack.understack_ref" $ }} + path: charts/nautobot-job-queues + helm: + ignoreMissingValueFiles: true + valueFiles: + - $deploy/{{ include "understack.deploy_path" $ }}/nautobot-job-queues/values.yaml + - path: {{ include "understack.deploy_path" $ }}/nautobot-job-queues + ref: deploy + repoURL: {{ include "understack.deploy_url" $ }} + targetRevision: {{ include "understack.deploy_ref" $ }} + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true + - RespectIgnoreDifferences=true + - ApplyOutOfSyncOnly=true +{{- end }} diff --git a/charts/argocd-understack/values.yaml b/charts/argocd-understack/values.yaml index 7cd752312..2f16bbce0 100644 --- a/charts/argocd-understack/values.yaml +++ b/charts/argocd-understack/values.yaml @@ -154,6 +154,12 @@ global: # @default -- false enabled: false + # -- Nautobot Celery JobQueue bootstrap jobs + nautobot_job_queues: + # -- Enable/disable deploying + # @default -- false + enabled: false + # -- Nautobot Operator for Kubernetes nautobotop: # -- Enable/disable deploying Nautobot Operator diff --git a/charts/nautobot-job-queues/Chart.yaml b/charts/nautobot-job-queues/Chart.yaml new file mode 100644 index 000000000..27965076d --- /dev/null +++ b/charts/nautobot-job-queues/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: nautobot-job-queues +description: Ensure Nautobot Celery JobQueue records from Helm values + +type: application + +version: 0.1.0 +# renovate: datasource=docker depName=networktocode/nautobot versioning=semver +appVersion: "3.0.7" + +maintainers: + - name: rackerlabs diff --git a/charts/nautobot-job-queues/ci/example.yaml b/charts/nautobot-job-queues/ci/example.yaml new file mode 100644 index 000000000..dd65c3ec3 --- /dev/null +++ b/charts/nautobot-job-queues/ci/example.yaml @@ -0,0 +1,2 @@ +queues: + - site-dc diff --git a/charts/nautobot-job-queues/templates/_helpers.tpl b/charts/nautobot-job-queues/templates/_helpers.tpl new file mode 100644 index 000000000..0d81e0b48 --- /dev/null +++ b/charts/nautobot-job-queues/templates/_helpers.tpl @@ -0,0 +1,49 @@ +{{/* Expand the name of the chart. */}} +{{- define "nautobot-job-queues.name" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* Create a default fully qualified app name. */}} +{{- define "nautobot-job-queues.fullname" -}} +{{- if contains .Chart.Name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* Chart label value. */}} +{{- define "nautobot-job-queues.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* Common labels. */}} +{{- define "nautobot-job-queues.labels" -}} +helm.sh/chart: {{ include "nautobot-job-queues.chart" . }} +{{ include "nautobot-job-queues.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* Selector labels. */}} +{{- define "nautobot-job-queues.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nautobot-job-queues.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* Script config map name. */}} +{{- define "nautobot-job-queues.scriptConfigMapName" -}} +{{- printf "%s-script" (include "nautobot-job-queues.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* Desired JobQueue config map name. */}} +{{- define "nautobot-job-queues.desiredConfigMapName" -}} +{{- printf "%s-desired" (include "nautobot-job-queues.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* Ensure job name. */}} +{{- define "nautobot-job-queues.jobName" -}} +{{- printf "%s-ensure" (include "nautobot-job-queues.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end }} diff --git a/charts/nautobot-job-queues/templates/configmap-desired-queues.yaml b/charts/nautobot-job-queues/templates/configmap-desired-queues.yaml new file mode 100644 index 000000000..253c855d5 --- /dev/null +++ b/charts/nautobot-job-queues/templates/configmap-desired-queues.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nautobot-job-queues.desiredConfigMapName" . }} + labels: + {{- include "nautobot-job-queues.labels" . | nindent 4 }} +data: + job-queues.json: | + {{ .Values.queues | toJson }} diff --git a/charts/nautobot-job-queues/templates/configmap-script.yaml b/charts/nautobot-job-queues/templates/configmap-script.yaml new file mode 100644 index 000000000..b0e62870e --- /dev/null +++ b/charts/nautobot-job-queues/templates/configmap-script.yaml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nautobot-job-queues.scriptConfigMapName" . }} + labels: + {{- include "nautobot-job-queues.labels" . | nindent 4 }} +data: + ensure.py: | + import json + + from django.apps import apps + + + def _load_desired(path): + with open(path, "r", encoding="utf-8") as fp: + raw = json.load(fp) + if not isinstance(raw, list): + raise RuntimeError(f"expected a JSON list in {path}") + for index, name in enumerate(raw): + if not isinstance(name, str) or not name: + raise RuntimeError(f"queue entry {index} must be a non-empty string") + return raw + + + JobQueue = apps.get_model("extras", "JobQueue") + + queue_type = "celery" + created = 0 + updated = 0 + unchanged = 0 + + for name in _load_desired("/desired/job-queues.json"): + job_queue = JobQueue.objects.filter(name=name).first() + + if job_queue is None: + job_queue = JobQueue.objects.create(name=name, queue_type=queue_type) + created += 1 + print(f"Created Nautobot JobQueue '{job_queue.name}' ({job_queue.queue_type})") + continue + + if job_queue.queue_type != queue_type: + old_queue_type = job_queue.queue_type + job_queue.queue_type = queue_type + job_queue.save(update_fields=["queue_type"]) + updated += 1 + print( + f"Updated Nautobot JobQueue '{job_queue.name}' queue_type " + f"from '{old_queue_type}' to '{job_queue.queue_type}'" + ) + continue + + unchanged += 1 + print(f"Nautobot JobQueue '{job_queue.name}' already exists") + + print( + "Nautobot JobQueue ensure complete: " + f"{created} created, {updated} updated, {unchanged} unchanged" + ) diff --git a/charts/nautobot-job-queues/templates/job.yaml b/charts/nautobot-job-queues/templates/job.yaml new file mode 100644 index 000000000..7dfec93c3 --- /dev/null +++ b/charts/nautobot-job-queues/templates/job.yaml @@ -0,0 +1,57 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "nautobot-job-queues.jobName" . }} + labels: + {{- include "nautobot-job-queues.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/hook: "PostSync" + argocd.argoproj.io/hook-delete-policy: "BeforeHookCreation" + argocd.argoproj.io/sync-wave: "10" +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 300 + template: + metadata: + labels: + {{- include "nautobot-job-queues.selectorLabels" . | nindent 8 }} + nautobot-job-queues/job: "ensure" + spec: + serviceAccountName: nautobot + restartPolicy: OnFailure + containers: + - name: ensure + image: {{ printf "ghcr.io/nautobot/nautobot:%s" .Chart.AppVersion | quote }} + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -lc + args: + - nautobot-server shell --interface python < /scripts/ensure.py + env: + - name: NAUTOBOT_DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.dbPasswordSecretRef.name | quote }} + key: {{ .Values.dbPasswordSecretRef.key | quote }} + envFrom: + - configMapRef: + name: nautobot-env + - secretRef: + name: nautobot-env + - secretRef: + name: nautobot-custom-env + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: desired + mountPath: /desired + readOnly: true + volumes: + - name: scripts + configMap: + name: {{ include "nautobot-job-queues.scriptConfigMapName" . }} + - name: desired + configMap: + name: {{ include "nautobot-job-queues.desiredConfigMapName" . }} diff --git a/charts/nautobot-job-queues/values.schema.json b/charts/nautobot-job-queues/values.schema.json new file mode 100644 index 000000000..6f504dc11 --- /dev/null +++ b/charts/nautobot-job-queues/values.schema.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "dbPasswordSecretRef": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "key": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "name", + "key" + ] + }, + "queues": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "dbPasswordSecretRef", + "queues" + ] +} diff --git a/charts/nautobot-job-queues/values.yaml b/charts/nautobot-job-queues/values.yaml new file mode 100644 index 000000000..727d51bbc --- /dev/null +++ b/charts/nautobot-job-queues/values.yaml @@ -0,0 +1,8 @@ +# -- Secret reference used to set NAUTOBOT_DB_PASSWORD on the managed job container. +dbPasswordSecretRef: + name: nautobot-env + key: NAUTOBOT_DB_PASSWORD + +# -- Nautobot JobQueue names to ensure. +queues: [] +# - site-dc diff --git a/docs/operator-guide/nautobot-celery-queues.md b/docs/operator-guide/nautobot-celery-queues.md index c42c26db2..c004fc6e9 100644 --- a/docs/operator-guide/nautobot-celery-queues.md +++ b/docs/operator-guide/nautobot-celery-queues.md @@ -61,54 +61,18 @@ Before any job can be dispatched to a site queue, a `JobQueue` record must exist in Nautobot's database. Without it, the API rejects the request with a validation error. -### Create via the UI - -Navigate to Jobs > Job Queues > Add and create a queue with: - -- Name: `site-dc` (must match the worker's `taskQueues` value) -- Queue Type: `celery` - -### Create via the REST API - -```bash -curl -X POST \ - -H "Authorization: Token $TOKEN" \ - -H "Content-Type: application/json" \ - https://nautobot.example.com/api/extras/job-queues/ \ - --data '{"name": "site-dc", "queue_type": "celery"}' -``` - -### Create via pynautobot - -```python -import pynautobot - -nb = pynautobot.api("https://nautobot.example.com", token="your-token") -nb.extras.job_queues.create(name="site-dc", queue_type="celery") -``` - -### Automate via Ansible - -The `ansible/roles/jobs/tasks/main.yml` role enables jobs but does not -currently create JobQueues. You can extend it: - -{% raw %} +For understack-managed environments, declare the desired queues in the +global deploy repo path: ```yaml -- name: "Ensure site JobQueue exists" - ansible.builtin.uri: - url: "{{ nautobot_url }}/api/extras/job-queues/" - method: POST - headers: - Authorization: "Token {{ nautobot_token }}" - body_format: json - body: - name: "{{ site }}" - queue_type: "celery" - status_code: [200, 201, 400] +# rax-dev-global/nautobot-job-queues/values.yaml +queues: + - site-dc ``` -{% endraw %} +The `nautobot-job-queues` ArgoCD application runs in the global +cluster and ensures these records exist in the shared Nautobot +database. ## Assigning Jobs to Queues