feat(runbooks): k3s + cloudshell-fog DevSecOps runbook library (#51)

Three default runbooks for the cloudshell-fog / k3s twin admin surface:

- k3s-cluster-admin: open tunnel, node/pod/event inspection, resource pressure
- cloudshell-devsecops: bastion SSH patterns, SOCKS5 proxy, remote exec, scp
- k3s-incident-triage: rapid blast-radius + pod failure + OOM + storage diagnosis

Searchable via `rb search k3s` or `rb search cloudshell`.

Author: mdheller

assets/sourceos/runbooks/cloudshell-devsecops.yaml

@@ -0,0 +1,24 @@
+name: cloudshell-devsecops
+description: "Cloudshell-fog DevSecOps session — bastion SSH, SOCKS5 proxy, remote exec patterns"
+tags: [cloudshell, devsecops, ssh, bastion, proxy, k3s]
+vars:
+  REMOTE_CMD: "kubectl get nodes"
+  SRC: "~/.kube/config-k3s-twin"
+  DST: "cloudshell:~/.kube/config-k3s-twin"
+steps:
+  - cmd: csh-status
+    desc: "Print full cloudshell-fog connection config and live tunnel/proxy status."
+  - cmd: csh-exec "whoami && hostname && uptime"
+    desc: "Non-interactive smoke-test — verify bastion reachability and identity."
+  - cmd: csh-exec "${REMOTE_CMD}"
+    desc: "Run an arbitrary remote command on the cloudshell bastion. Edit REMOTE_CMD var to taste."
+  - cmd: ktunnel start
+    desc: "Forward k3s API port (localhost:16443 → k3s-twin:6443) through the bastion."
+  - cmd: kproxy start
+    desc: "Open SOCKS5 proxy on localhost:1080 via cloudshell — routes sovereign browsing through the fog network."
+  - cmd: csh-copy "${SRC}" "${DST}"
+    desc: "Copy kubeconfig or any file to/from cloudshell via scp. Set SRC/DST vars: prefix 'cloudshell:' for remote side."
+  - cmd: csh
+    desc: "Drop into a full interactive SSH session on the cloudshell-fog bastion (CMD+SHIFT+K)."
+  - cmd: ktunnel stop
+    desc: "Tear down the k3s API tunnel cleanly when done."

assets/sourceos/runbooks/k3s-cluster-admin.yaml

@@ -0,0 +1,25 @@
+name: k3s-cluster-admin
+description: "k3s twin cluster admin — open tunnel, verify connectivity, inspect workloads"
+tags: [k3s, devsecops, cloudshell, tunnel, kubectl]
+vars:
+  NAMESPACE: "kube-system"
+  KUBECONFIG: "${CLOUDSHELL_K3S_KUBECONFIG:-$HOME/.kube/config-k3s-twin}"
+steps:
+  - cmd: csh-status
+    desc: "Show cloudshell-fog config and current tunnel/proxy state."
+  - cmd: ktunnel start
+    desc: "Open k3s API tunnel: localhost:16443 → k3s-twin:6443 via cloudshell-fog bastion. Safe to re-run if already up."
+  - cmd: k3s get nodes -o wide
+    desc: "Verify k3s twin reachability and node readiness via the tunnel."
+  - cmd: k3s get pods -n ${NAMESPACE} --field-selector='status.phase!=Running'
+    desc: "Surface any pods that are NOT running in the target namespace — the first triage signal."
+  - cmd: k3s get events -n ${NAMESPACE} --sort-by='.metadata.creationTimestamp' | tail -30
+    desc: "Last 30 cluster events sorted by timestamp — quick anomaly scan."
+  - cmd: k3s top nodes
+    desc: "CPU/memory pressure across nodes (requires metrics-server)."
+  - cmd: k3s top pods -n ${NAMESPACE} --sort-by=memory
+    desc: "Top pods by memory in the target namespace."
+  - cmd: k3s get all -A --show-labels
+    desc: "Full cluster inventory with labels — useful for verifying GitOps-applied state."
+  - cmd: ktunnel status
+    desc: "Confirm tunnel is still alive after the admin session."

assets/sourceos/runbooks/k3s-incident-triage.yaml

@@ -0,0 +1,27 @@
+name: k3s-incident-triage
+description: "k3s twin incident triage — rapid diagnosis for pod failures, OOM, crashloops, storage"
+tags: [k3s, devsecops, incident, triage, cloudshell]
+vars:
+  NAMESPACE: "default"
+  POD: ""
+steps:
+  - cmd: ktunnel start
+    desc: "Ensure k3s API tunnel is open. Safe to re-run."
+  - cmd: k3s get pods -A --field-selector='status.phase!=Running' -o wide
+    desc: "All non-running pods across every namespace — the incident blast radius at a glance."
+  - cmd: k3s get events -A --field-selector='type=Warning' --sort-by='.metadata.creationTimestamp' | tail -40
+    desc: "Warning events cluster-wide, newest last."
+  - cmd: k3s describe pod ${POD} -n ${NAMESPACE}
+    desc: "Full pod description for the affected pod. Set POD var to the pod name."
+  - cmd: k3s logs ${POD} -n ${NAMESPACE} --previous --tail=100
+    desc: "Logs from the previous (crashed) container instance — the real error trace."
+  - cmd: k3s logs ${POD} -n ${NAMESPACE} --tail=100
+    desc: "Logs from the current container instance."
+  - cmd: k3s get pvc -n ${NAMESPACE}
+    desc: "PVC status in the affected namespace — catch Pending volumes blocking pod scheduling."
+  - cmd: k3s describe node $(k3s get pods ${POD} -n ${NAMESPACE} -o jsonpath='{.spec.nodeName}')
+    desc: "Node conditions and allocatable resources for the pod's assigned node."
+  - cmd: csh-exec "journalctl -u k3s --since '30 minutes ago' | tail -60"
+    desc: "k3s systemd service journal on the cloudshell-fog host — catches API server and etcd panics."
+  - cmd: k3s rollout restart deployment/${POD%-*} -n ${NAMESPACE}
+    desc: "Controlled rollout restart of the affected deployment. Only run after diagnosing root cause."