From baf3656660f6d3dcb53f2f7d77b2db02c6cc577a Mon Sep 17 00:00:00 2001 From: wwttest <167965921+wwthw@users.noreply.github.com> Date: Thu, 25 Jun 2026 08:46:01 +0000 Subject: [PATCH 1/2] CubeSandbox on Kubernetes, compute node image build and deploy. --- .../images/cubelet-network-agent/Dockerfile | 85 ++++++++ .../cubelet-network-agent/start-cubelet.sh | 59 +++++ .../start-network-agent.sh | 52 +++++ .../manifests/configmaps/cubelet-config.yaml | 204 ++++++++++++++++++ .../cubelet-dynamicconf-control.yaml | 35 +++ .../manifests/daemonsets/cubelet-control.yaml | 192 +++++++++++++++++ deploy/k8s/manifests/namespace.yaml | 8 + .../build-cubelet-network-agent-image.sh | 183 ++++++++++++++++ 8 files changed, 818 insertions(+) create mode 100644 deploy/k8s/images/cubelet-network-agent/Dockerfile create mode 100644 deploy/k8s/images/cubelet-network-agent/start-cubelet.sh create mode 100644 deploy/k8s/images/cubelet-network-agent/start-network-agent.sh create mode 100644 deploy/k8s/manifests/configmaps/cubelet-config.yaml create mode 100644 deploy/k8s/manifests/configmaps/cubelet-dynamicconf-control.yaml create mode 100644 deploy/k8s/manifests/daemonsets/cubelet-control.yaml create mode 100644 deploy/k8s/manifests/namespace.yaml create mode 100644 deploy/k8s/scripts/build-cubelet-network-agent-image.sh diff --git a/deploy/k8s/images/cubelet-network-agent/Dockerfile b/deploy/k8s/images/cubelet-network-agent/Dockerfile new file mode 100644 index 00000000..19c071d0 --- /dev/null +++ b/deploy/k8s/images/cubelet-network-agent/Dockerfile @@ -0,0 +1,85 @@ +# Cubelet + Network-Agent Combined Image + +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + e2fsprogs \ + util-linux \ + uidmap \ + bash \ + coreutils \ + procps \ + iputils-ping \ + iproute2 \ + tcpdump \ + linux-tools-common \ + linux-tools-generic \ + iperf3 \ + net-tools \ + bridge-utils \ + ethtool \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p \ + /usr/local/bin \ + /usr/local/services/cubetoolbox/Cubelet/bin \ + /usr/local/services/cubetoolbox/Cubelet/config \ + /usr/local/services/cubetoolbox/Cubelet/dynamicconf \ + /usr/local/services/cubetoolbox/network-agent/bin \ + /usr/local/services/cubetoolbox/network-agent/state \ + /usr/local/services/cubetoolbox/cube-shim/bin \ + /usr/local/services/cubetoolbox/cube-shim/conf \ + /usr/local/services/cubetoolbox/cube-hypervisor \ + /usr/local/services/cubetoolbox/cube-snapshot \ + /data/cubelet \ + /data/log/Cubelet \ + /data/log/CubeShim \ + /data/log/CubeVmm \ + /data/cube-shim/disks \ + /data/snapshot_pack/disks \ + /run/cube-containers/shared/sandboxes \ + /tmp/cube + + +COPY cubelet /usr/local/services/cubetoolbox/Cubelet/bin/cubelet +COPY cubecli /usr/local/services/cubetoolbox/Cubelet/bin/cubecli +COPY nicl /usr/local/services/cubetoolbox/Cubelet/bin/nicl +COPY cubelet-code-deploy.sh /usr/local/services/cubetoolbox/Cubelet/bin/cubelet-code-deploy.sh +COPY unsquashfs /usr/local/services/cubetoolbox/Cubelet/bin/unsquashfs +COPY unsquashfs-dio /usr/local/services/cubetoolbox/Cubelet/bin/unsquashfs-dio +COPY containerd-shim-cube-rs /usr/local/services/cubetoolbox/cube-shim/bin/containerd-shim-cube-rs +COPY cube-runtime /usr/local/services/cubetoolbox/cube-shim/bin/cube-runtime +COPY network-agent /usr/local/services/cubetoolbox/network-agent/bin/network-agent +COPY config-cube.toml /usr/local/services/cubetoolbox/cube-shim/conf/config-cube.toml +COPY network-agent.yaml /usr/local/services/cubetoolbox/network-agent/network-agent.yaml +COPY snapshot.sh /usr/local/services/cubetoolbox/Cubelet/config/snapshot.sh + + +RUN chmod +x \ + /usr/local/services/cubetoolbox/Cubelet/bin/cubelet \ + /usr/local/services/cubetoolbox/Cubelet/bin/cubecli \ + /usr/local/services/cubetoolbox/Cubelet/bin/nicl \ + /usr/local/services/cubetoolbox/Cubelet/bin/cubelet-code-deploy.sh \ + /usr/local/services/cubetoolbox/Cubelet/bin/unsquashfs \ + /usr/local/services/cubetoolbox/Cubelet/bin/unsquashfs-dio \ + /usr/local/services/cubetoolbox/cube-shim/bin/containerd-shim-cube-rs \ + /usr/local/services/cubetoolbox/cube-shim/bin/cube-runtime \ + /usr/local/services/cubetoolbox/network-agent/bin/network-agent + +COPY start-network-agent.sh /usr/local/bin/start-network-agent.sh +COPY start-cubelet.sh /usr/local/bin/start-cubelet.sh +RUN chmod +x /usr/local/bin/start-network-agent.sh /usr/local/bin/start-cubelet.sh + +RUN ln -sf /usr/local/services/cubetoolbox/cube-shim/bin/cube-runtime /usr/local/bin/cube-runtime +RUN ln -sf /usr/local/services/cubetoolbox/cube-shim/bin/containerd-shim-cube-rs /usr/local/bin/containerd-shim-cube-rs + +RUN chmod +x /usr/local/services/cubetoolbox/Cubelet/config/snapshot.sh + +WORKDIR /data/cubelet + +ENTRYPOINT ["/usr/local/bin/start-cubelet.sh"] +CMD ["--config", "/usr/local/services/cubetoolbox/Cubelet/config/config.toml"] \ No newline at end of file diff --git a/deploy/k8s/images/cubelet-network-agent/start-cubelet.sh b/deploy/k8s/images/cubelet-network-agent/start-cubelet.sh new file mode 100644 index 00000000..0ae34c34 --- /dev/null +++ b/deploy/k8s/images/cubelet-network-agent/start-cubelet.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -euo pipefail + +# Cubelet start +# Keep the file paths consistent with the one-click deployment configuration. + + +export PATH="/usr/local/services/cubetoolbox/Cubelet/bin:${PATH}" + +CUBELET_BIN="${CUBELET_BIN:-/usr/local/services/cubetoolbox/Cubelet/bin/cubelet}" +CUBELET_CONFIG="${CUBELET_CONFIG:-/usr/local/services/cubetoolbox/Cubelet/config/config.toml}" +DYNAMIC_CONF="${DYNAMIC_CONF:-/usr/local/services/cubetoolbox/Cubelet/dynamicconf/conf.yaml}" +NETWORK_AGENT_SOCKET="${NETWORK_AGENT_SOCKET:-/tmp/cube/network-agent-grpc.sock}" +NETWORK_AGENT_HEALTH="${NETWORK_AGENT_HEALTH:-127.0.0.1:19090}" + +echo "Starting cubelet..." +echo " CUBELET_BIN: ${CUBELET_BIN}" +echo " CUBELET_CONFIG: ${CUBELET_CONFIG}" +echo " DYNAMIC_CONF: ${DYNAMIC_CONF}" +echo " NETWORK_AGENT_SOCKET: ${NETWORK_AGENT_SOCKET}" +echo " NETWORK_AGENT_HEALTH: ${NETWORK_AGENT_HEALTH}" +echo " PATH: ${PATH}" + +if [[ ! -S "${NETWORK_AGENT_SOCKET}" ]]; then + echo "Waiting for network-agent socket: ${NETWORK_AGENT_SOCKET}" + for i in {1..60}; do + if [[ -S "${NETWORK_AGENT_SOCKET}" ]]; then + echo "network-agent socket ready" + break + fi + sleep 1 + done + if [[ ! -S "${NETWORK_AGENT_SOCKET}" ]]; then + echo "ERROR: network-agent socket not found after 60s" + exit 1 + fi +fi + +echo "Checking network-agent health..." +for i in {1..30}; do + if curl -fsS "http://${NETWORK_AGENT_HEALTH}/healthz" >/dev/null 2>&1; then + echo "network-agent health check passed" + break + fi + sleep 1 +done + + +if mountpoint -q /sys/fs/bpf; then + umount /sys/fs/bpf +fi +mkdir -p /sys/fs/bpf +mount -t bpf bpf /sys/fs/bpf -o mode=0700 +echo "Mounted private bpffs at /sys/fs/bpf" + +# start cubelet +exec "${CUBELET_BIN}" \ + --config "${CUBELET_CONFIG}" \ + --dynamic-conf-path "${DYNAMIC_CONF}" \ No newline at end of file diff --git a/deploy/k8s/images/cubelet-network-agent/start-network-agent.sh b/deploy/k8s/images/cubelet-network-agent/start-network-agent.sh new file mode 100644 index 00000000..ca076097 --- /dev/null +++ b/deploy/k8s/images/cubelet-network-agent/start-network-agent.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -euo pipefail + +# Network-Agent start +# Keep the file paths consistent with the one-click deployment configuration. + +NETWORK_AGENT_BIN="${NETWORK_AGENT_BIN:-/usr/local/services/cubetoolbox/network-agent/bin/network-agent}" +CUBELET_CONFIG="${CUBELET_CONFIG:-/usr/local/services/cubetoolbox/Cubelet/config/config.toml}" +GRPC_LISTEN="${GRPC_LISTEN:-unix:///tmp/cube/network-agent-grpc.sock}" +TAP_FD_LISTEN="${TAP_FD_LISTEN:-unix:///tmp/cube/network-agent-tap.sock}" +STATE_DIR="${STATE_DIR:-/usr/local/services/cubetoolbox/network-agent/state}" +HEALTH_LISTEN="${HEALTH_LISTEN:-127.0.0.1:19090}" + +echo "Starting network-agent..." +echo " NETWORK_AGENT_BIN: ${NETWORK_AGENT_BIN}" +echo " CUBELET_CONFIG: ${CUBELET_CONFIG}" +echo " GRPC_LISTEN: ${GRPC_LISTEN}" +echo " TAP_FD_LISTEN: ${TAP_FD_LISTEN}" +echo " STATE_DIR: ${STATE_DIR}" +echo " HEALTH_LISTEN: ${HEALTH_LISTEN}" + +if [[ ! -f "${CUBELET_CONFIG}" ]]; then + echo "Waiting for cubelet config: ${CUBELET_CONFIG}" + for i in {1..30}; do + if [[ -f "${CUBELET_CONFIG}" ]]; then + break + fi + sleep 1 + done + if [[ ! -f "${CUBELET_CONFIG}" ]]; then + echo "ERROR: cubelet config not found after 30s" + exit 1 + fi +fi + +mkdir -p "${STATE_DIR}" +mkdir -p /tmp/cube + +if mountpoint -q /sys/fs/bpf; then + umount /sys/fs/bpf +fi +mkdir -p /sys/fs/bpf +mount -t bpf bpf /sys/fs/bpf -o mode=0700 +echo "Mounted private bpffs at /sys/fs/bpf" + +# start network-agent +exec "${NETWORK_AGENT_BIN}" \ + --cubelet-config "${CUBELET_CONFIG}" \ + --grpc-listen "${GRPC_LISTEN}" \ + --tap-fd-listen "${TAP_FD_LISTEN}" \ + --state-dir "${STATE_DIR}" \ + --health-listen "${HEALTH_LISTEN}" \ No newline at end of file diff --git a/deploy/k8s/manifests/configmaps/cubelet-config.yaml b/deploy/k8s/manifests/configmaps/cubelet-config.yaml new file mode 100644 index 00000000..98ab671a --- /dev/null +++ b/deploy/k8s/manifests/configmaps/cubelet-config.yaml @@ -0,0 +1,204 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cubelet-config + namespace: cube-system +data: + config.toml: | + oom_score = 0 + root = "/data/cubelet/root" + state = "/data/cubelet/state" + version = 3 + pid_file = "/run/cube-let.pid" + dynamic_config_path = "/usr/local/services/cubetoolbox/Cubelet/dynamicconf/conf.yaml" + + [http] + address = ":9998" + + [timeouts] + "io.containerd.timeout.bolt.open" = "3s" + "io.containerd.timeout.task.state" = "2s" + "io.containerd.timeout.shim.shutdown" = "5m" + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + + [grpc] + address = "/data/cubelet/cubelet.sock" + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + tcp_address = ":9999" + uid = 0 + + [cubetap] + address = "/data/cubelet/cubetap.sock" + gid = 0 + uid = 0 + + [operation_server] + address = "/data/cubelet/cubelet-operation.sock" + gid = 0 + uid = 0 + disable = true + + [debug] + address = ":9966" + + [plugins] + [plugins."io.cubelet.controller.config.v1.cubelet"] + # Static-only cadence for node status and resource reporting. + # Do not configure this in dynamicconf/conf.yaml. + node_status_update_frequency = "10s" + + [plugins."io.cubelet.internal.v1.volume"] + + [plugins."io.cubelet.internal.v1.shimlog"] + root_path = "/data/cubelet/shimlog" + + [plugins."io.cubelet.internal.v1.cleanup"] + root_path = "/data/cubelet/cleanup" + + [plugins."io.cubelet.internal.v1.cgroup"] + pool_size = 3000 + pool_workers = 1 + pool_trigger_interval_in_ms = 1000 + vm_cpu_overhead = "0" + vm_memory_overhead_base = "42Mi" + vm_memory_overhead_coefficient = 64 + host_cpu_overhead = "0.3" + host_memory_overhead_base = "20Mi" + vm_snapshot_specs_config = "/usr/local/services/cubetoolbox/cube-snapshot/spec.json" + snapshot_disk_dir = "/data/snapshot_pack/disks" + + [plugins."io.cubelet.internal.v1.network"] + object_dir = "/usr/local/services/cubetoolbox/cube-vs/network" + eth_name = "eth0" + tap_init_num = 500 + cidr = "192.168.0.0/18" + mvm_inner_ip = "169.254.68.6" + mvm_mac_addr = "20:90:6f:fc:fc:fc" + mvm_gw_mac_addr = "20:90:6f:cf:cf:cf" + mvm_gw_dest_ip = "169.254.68.5" + mvm_mtu = 1300 + disable_tso = true + disable_ufo = true + disable_check_sum = true + check_interval_in_sec = "5s" + report_stat_interval_in_sec = "60s" + watch_stream = false + redis_conf_path = "/data/cubelet/rainbow.toml" + stream_name_prefix = "gw_route:update:" + stream_key = "gw_key" + stream_block_time = "30s" + enable_network_agent = true + network_agent_endpoint = "grpc+unix:///tmp/cube/network-agent-grpc.sock" + network_agent_tap_socket = "/tmp/cube/network-agent-tap.sock" + network_agent_init_timeout = "120s" + network_agent_retry_interval = "1s" + network_agent_tap_fd_timeout = "2s" + + [plugins."io.cubelet.internal.v1.storage"] + # Default storage backend: cubecow (reflink-only copy-on-write). + storage_backend = "cubecow" + + # Working directory for the cubelet storage plugin. cubecow reflink volumes land at + # /cubecow-reflink, so the filesystem backing data_path must support + # FICLONE (e.g. xfs with `-m reflink=1` or btrfs). + data_path = "/data/cubelet/storage" + + + [plugins."io.cubelet.internal.v1.storage".cow.log] + level = "info,h2=warn,hyper=warn,tower=warn" + format = "compact" + file = "/data/log/cubecow/cubecow.log" + rotation = "daily" + + # Only needed when switching back to the legacy ext4 / raw / reflink-pool + # backend: change storage_backend to the corresponding value (e.g. "ext4") + # and uncomment the fields below as needed. Under cubecow these fields are ignored. + # pool_type = "copy_reflink" + # disksize = "400Gi" + # warningPercent = 200 + # pool_default_format_size_list = ["1Gi"] + # base_disk_uuid = "ef5c2893-ddbd-4d6e-bef6-3853c31d5b94" + # pool_size = 3000 + # pool_worker_num = 8 + # pool_trigger_interval_in_ms = 800 + + [plugins."io.cubelet.internal.v1.images"] + runtime_type = "io.containerd.cube.v2" + + [plugins."io.cubelet.internal.v1.cubebox"] + default_runtime_name = "cube" + cube_shim_path = "/usr/local/services/cubetoolbox/cube-shim/bin/containerd-shim-cube-rs" + cube_hypervisor_path = "/usr/local/services/cubetoolbox/cube-hypervisor/cube-hypervisor" + + [plugins."io.cubelet.internal.v1.cubebox".runtimes] + [plugins."io.cubelet.internal.v1.cubebox".runtimes.cube] + runtime_type = "io.containerd.cube.rs" + runtime_cfg_path = "/usr/local/services/cubetoolbox/cube-shim/conf/config-cube.toml" + [plugins."io.cubelet.internal.v1.cubebox".runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + cubetool_base_dir = "/usr/local/services/cubetoolbox" + + [plugins."io.cubelet.cubebox-service.v1.cubebox-service"] + destroy_dead_line = "60s" + dead_container_ttl = "1h" + + [plugins."io.cubelet.images-service.v1.images-service"] + [plugins."io.cubelet.images-service.v1.images-service".image_gc] + max_unused_time_interval = "1d" + detection_interval = "1m" + max_deletion_per_cycle = 10 + free_disk_threshold_percent = 10 + + [plugins."io.cubelet.internal.v1.gc-service"] + gc_interval = "30s" + cleanup_interval = "5s" + + [plugins."io.cubelet.workflow.v1.workflow"] + [plugins."io.cubelet.workflow.v1.workflow".flows] + [plugins."io.cubelet.workflow.v1.workflow".flows.init] + actions = [["cleanup"],["cubebox"],["images","storage","cgroup","network","volume","netfile","cube-sandbox-store"]] + [plugins."io.cubelet.workflow.v1.workflow".flows.create] + concurrent = 100 + actions = [["createid","appsnapshot"],["images","volume","storage","network","netfile","cube-sandbox-store"],["cgroup"],["cubebox"]] + [plugins."io.cubelet.workflow.v1.workflow".flows.destroy] + concurrent = 100 + actions = [["cubebox"],["images","storage","cgroup","network","volume","netfile","cube-sandbox-store"],["cleanup"]] + [plugins."io.cubelet.workflow.v1.workflow".flows.cleanup] + actions = [["cubebox"],["images","volume","storage","cgroup","network","netfile","cube-sandbox-store"],["cleanup"]] + + [plugins."io.containerd.gc.v1.scheduler"] + deletion_threshold = 0 + mutation_threshold = 1000 + pause_threshold = 0.0001 + schedule_delay = "0s" + startup_delay = "5s" + + [plugins."io.containerd.snapshotter.v1.overlayfs"] + root_path = "/data/cubelet/state" + data_path = "/data/cubelet/root" + + [plugins."io.containerd.metadata.v1.bolt"] + root_path = "/data/cubelet/state" + content_sharing_policy = "shared" + no_sync = true + + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + sched_core = false + + [plugins."io.cubelet.chi.v1.vsocket-manager"] + proxyPort = 1032 + + [plugins."io.containerd.cri.v1.images"] + max_concurrent_downloads = 10 + stats_collect_period = 30 + [plugins."io.containerd.cri.v1.images".registry.mirrors] + [plugins."io.containerd.cri.v1.images".registry.mirrors."docker.io"] + endpoint = ["https://mirror.ccs.tencentyun.com"] \ No newline at end of file diff --git a/deploy/k8s/manifests/configmaps/cubelet-dynamicconf-control.yaml b/deploy/k8s/manifests/configmaps/cubelet-dynamicconf-control.yaml new file mode 100644 index 00000000..7859f6e2 --- /dev/null +++ b/deploy/k8s/manifests/configmaps/cubelet-dynamicconf-control.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cubelet-dynamicconf-control + namespace: cube-system +data: + conf.yaml: | + common: + enable_pf_mode: false + sandbox_exec_cmd_time_out: 5s + enable_sandbox_exec_cmd_before_exist: false + sandbox_exec_cmd_before_exist_log_out: false + sandbox_exec_cmd_before_exist: + - sh + - "/usr/local/cubetools/cube_exec_cmd_before_exist.sh" + cgroup_set_memory_reparent_file: "" + disable_host_cgroup: true + disable_host_netfile: true + default_dns_servers: + - 169.254.254.53 + + host: + scheduler_label: "default-cluster" + quota: + mcpu_limit: + mem_limit: "" + mvm_limit: 100 + creation_concurrent_num: 10 + gc: + code_expiration_time: "72h" + image_expiration_time: "24h" + + meta_server_config: + meta_server_endpoint: "cubemaster.cube-system.svc.cluster.local:8089" + node_status_max_images: 40000 \ No newline at end of file diff --git a/deploy/k8s/manifests/daemonsets/cubelet-control.yaml b/deploy/k8s/manifests/daemonsets/cubelet-control.yaml new file mode 100644 index 00000000..2071a8c0 --- /dev/null +++ b/deploy/k8s/manifests/daemonsets/cubelet-control.yaml @@ -0,0 +1,192 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cubelet-control + namespace: cube-system + labels: + app: cubelet-control + cube-role: control +spec: + selector: + matchLabels: + app: cubelet-control + template: + metadata: + labels: + app: cubelet-control + cube-role: control + spec: + nodeSelector: + cube-role: control + containers: + # container 1: network-agent + - name: network-agent + image: cube-sandbox/cubelet-network-agent:latest + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + ports: + - containerPort: 19090 + name: health + hostPort: 19090 + env: + - name: NETWORK_AGENT_BIN + value: "/usr/local/services/cubetoolbox/network-agent/bin/network-agent" + - name: CUBELET_CONFIG + value: "/usr/local/services/cubetoolbox/Cubelet/config/config.toml" + - name: GRPC_LISTEN + value: "unix:///tmp/cube/network-agent-grpc.sock" + - name: TAP_FD_LISTEN + value: "unix:///tmp/cube/network-agent-tap.sock" + - name: STATE_DIR + value: "/var/lib/network-agent/state" + - name: HEALTH_LISTEN + value: "127.0.0.1:19090" + command: + - /usr/local/bin/start-network-agent.sh + volumeMounts: + - name: tmp-cube + mountPath: /tmp/cube + - name: cubelet-config + mountPath: /usr/local/services/cubetoolbox/Cubelet/config + - name: network-agent-state + mountPath: /usr/local/services/cubetoolbox/network-agent/state + - name: network-agent-logs + mountPath: /data/log/network-agent + - name: dev-net-tun + mountPath: /dev/net/tun + resources: + limits: + cpu: "2" + memory: "4Gi" + requests: + cpu: "1" + memory: "2Gi" + + # container 2: cubelet (dependency network-agent) + - name: cubelet + image: cube-sandbox/cubelet-network-agent:0.0.2 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: CUBELET_BIN + value: "/usr/local/services/cubetoolbox/Cubelet/bin/cubelet" + - name: CUBELET_CONFIG + value: "/usr/local/services/cubetoolbox/Cubelet/config/config.toml" + - name: DYNAMIC_CONF + value: "/usr/local/services/cubetoolbox/Cubelet/dynamicconf/conf.yaml" + - name: NETWORK_AGENT_SOCKET + value: "/tmp/cube/network-agent-grpc.sock" + - name: NETWORK_AGENT_HEALTH + value: "127.0.0.1:19090" + command: + - /usr/local/bin/start-cubelet.sh + volumeMounts: + - name: tmp-cube + mountPath: /tmp/cube + - name: cubelet-config + mountPath: /usr/local/services/cubetoolbox/Cubelet/config + - name: cubelet-dynamicconf + mountPath: /usr/local/services/cubetoolbox/Cubelet/dynamicconf + - name: cubelet-data + mountPath: /data/cubelet + - name: dev-kvm + mountPath: /dev/kvm + - name: dev-net-tun + mountPath: /dev/net/tun + - name: cubelet-logs + mountPath: /data/log/Cubelet + - name: cube-shim-logs + mountPath: /data/log/CubeShim + - name: cube-vmm-logs + mountPath: /data/log/CubeVmm + - name: cube-shim-disks + mountPath: /data/cube-shim/disks + - name: cube-snapshot-disks + mountPath: /data/snapshot_pack/disks + - name: run-cube-containers + mountPath: /run/cube-containers + # VM assets + - name: cube-kernel-scf + mountPath: /usr/local/services/cubetoolbox/cube-kernel-scf + - name: cube-image + mountPath: /usr/local/services/cubetoolbox/cube-image + resources: + limits: + cpu: "4" + memory: "8Gi" + requests: + cpu: "2" + memory: "4Gi" + + volumes: + - name: tmp-cube + hostPath: + path: /tmp/cube + type: DirectoryOrCreate + - name: cubelet-config + configMap: + name: cubelet-config + items: + - key: config.toml + path: config.toml + - name: cubelet-dynamicconf + configMap: + name: cubelet-dynamicconf-control + items: + - key: conf.yaml + path: conf.yaml + - name: cubelet-data + hostPath: + path: /data/cubelet + type: DirectoryOrCreate + - name: network-agent-state + hostPath: + path: /usr/local/services/cubetoolbox/network-agent/state + type: DirectoryOrCreate + - name: dev-kvm + hostPath: + path: /dev/kvm + - name: dev-net-tun + hostPath: + path: /dev/net/tun + - name: cubelet-logs + hostPath: + path: /data/log/Cubelet + type: DirectoryOrCreate + - name: network-agent-logs + hostPath: + path: /data/log/network-agent + type: DirectoryOrCreate + - name: cube-shim-logs + hostPath: + path: /data/log/CubeShim + type: DirectoryOrCreate + - name: cube-vmm-logs + hostPath: + path: /data/log/CubeVmm + type: DirectoryOrCreate + - name: cube-shim-disks + hostPath: + path: /data/cube-shim/disks + type: DirectoryOrCreate + - name: cube-snapshot-disks + hostPath: + path: /data/snapshot_pack/disks + type: DirectoryOrCreate + - name: run-cube-containers + hostPath: + path: /run/cube-containers + type: DirectoryOrCreate + # VM assets + - name: cube-kernel-scf + hostPath: + path: /usr/local/services/cubetoolbox/cube-kernel-scf + type: Directory + - name: cube-image + hostPath: + path: /usr/local/services/cubetoolbox/cube-image + type: Directory + + priorityClassName: system-node-critical \ No newline at end of file diff --git a/deploy/k8s/manifests/namespace.yaml b/deploy/k8s/manifests/namespace.yaml new file mode 100644 index 00000000..74314f4d --- /dev/null +++ b/deploy/k8s/manifests/namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cube-system + labels: + name: cube-system + app.kubernetes.io/name: cube-sandbox + app.kubernetes.io/component: infrastructure \ No newline at end of file diff --git a/deploy/k8s/scripts/build-cubelet-network-agent-image.sh b/deploy/k8s/scripts/build-cubelet-network-agent-image.sh new file mode 100644 index 00000000..e6f399b4 --- /dev/null +++ b/deploy/k8s/scripts/build-cubelet-network-agent-image.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Build Cubelet + Network-Agent Image + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +IMAGE_DIR="${SCRIPT_DIR}/../images/cubelet-network-agent" + +IMAGE_REGISTRY="${IMAGE_REGISTRY:-cube-sandbox}" +IMAGE_TAG="${IMAGE_TAG:-$(git rev-parse --short HEAD)}" +PREBUILT_DIR="${PREBUILT_DIR:-${ROOT_DIR}/deploy/one-click/.work/prebuilt}" + +echo "======================================" +echo "Building Cubelet-Network-Agent Image" +echo "======================================" +echo "Registry: ${IMAGE_REGISTRY}" +echo "Tag: ${IMAGE_TAG}" +echo "Prebuilt Dir: ${PREBUILT_DIR}" +echo "Image Dir: ${IMAGE_DIR}" +echo "" + + +check_prebuilt_binaries() { + local binaries=( + "cubelet" + "cubecli" + "containerd-shim-cube-rs" + "cube-runtime" + "network-agent" + ) + + echo "Checking prebuilt binaries..." + for bin in "${binaries[@]}"; do + local path="${PREBUILT_DIR}/${bin}" + if [[ ! -f "${path}" ]]; then + echo "ERROR: Binary not found: ${path}" + echo "Please run build-release-bundle-builder.sh first" + exit 1 + fi + echo " ✓ ${bin} ($(du -h "${path}" | cut -f1))" + done + echo "" + echo "Note: containerd-shim-cube-rs includes hypervisor library (lib_support)" + echo "" +} + +# Preparing build context +prepare_build_context() { + echo "Preparing build context..." + + cp "${PREBUILT_DIR}/cubelet" "${IMAGE_DIR}/cubelet" + cp "${PREBUILT_DIR}/cubecli" "${IMAGE_DIR}/cubecli" + + if [[ -f "${ROOT_DIR}/Cubelet/contrib/nicl" ]]; then + cp "${ROOT_DIR}/Cubelet/contrib/nicl" "${IMAGE_DIR}/nicl" + chmod +x "${IMAGE_DIR}/nicl" + fi + + if [[ -f "${ROOT_DIR}/Cubelet/contrib/cubelet-code-deploy.sh" ]]; then + cp "${ROOT_DIR}/Cubelet/contrib/cubelet-code-deploy.sh" "${IMAGE_DIR}/cubelet-code-deploy.sh" + chmod +x "${IMAGE_DIR}/cubelet-code-deploy.sh" + fi + + if [[ -f "${ROOT_DIR}/Cubelet/contrib/unsquashfs" ]]; then + cp "${ROOT_DIR}/Cubelet/contrib/unsquashfs" "${IMAGE_DIR}/unsquashfs" + chmod +x "${IMAGE_DIR}/unsquashfs" + else + echo "WARNING: unsquashfs not found in Cubelet/contrib/" + fi + + if [[ -f "${ROOT_DIR}/Cubelet/contrib/unsquashfs-dio" ]]; then + cp "${ROOT_DIR}/Cubelet/contrib/unsquashfs-dio" "${IMAGE_DIR}/unsquashfs-dio" + chmod +x "${IMAGE_DIR}/unsquashfs-dio" + else + echo "WARNING: unsquashfs-dio not found in Cubelet/contrib/" + fi + + cp "${PREBUILT_DIR}/containerd-shim-cube-rs" "${IMAGE_DIR}/containerd-shim-cube-rs" + cp "${PREBUILT_DIR}/cube-runtime" "${IMAGE_DIR}/cube-runtime" + cp "${PREBUILT_DIR}/network-agent" "${IMAGE_DIR}/network-agent" + cp "${ROOT_DIR}/deploy/one-click/config-cube.toml" "${IMAGE_DIR}/config-cube.toml" + cp "${ROOT_DIR}/configs/single-node/network-agent.yaml" "${IMAGE_DIR}/network-agent.yaml" + + if [[ -f "${ROOT_DIR}/Cubelet/config/snapshot.sh" ]]; then + cp "${ROOT_DIR}/Cubelet/config/snapshot.sh" "${IMAGE_DIR}/snapshot.sh" + chmod +x "${IMAGE_DIR}/snapshot.sh" + fi + + echo " ✓ All files copied to ${IMAGE_DIR}" + echo "" +} + +# Building Docker image +build_image() { + echo "Building Docker image..." + + docker build \ + -t "${IMAGE_REGISTRY}/cubelet-network-agent:${IMAGE_TAG}" \ + -t "${IMAGE_REGISTRY}/cubelet-network-agent:latest" \ + -f "${IMAGE_DIR}/Dockerfile" \ + "${IMAGE_DIR}" + + echo " ✓ Image built successfully" + echo "" +} + +# Push image to registry Registry +push_image() { + if [[ "${PUSH_IMAGE:-false}" == "true" ]]; then + echo "Pushing image to registry..." + docker push "${IMAGE_REGISTRY}/cubelet-network-agent:${IMAGE_TAG}" + docker push "${IMAGE_REGISTRY}/cubelet-network-agent:latest" + echo " ✓ Image pushed successfully" + echo "" + fi +} + +# cleanup +cleanup() { + echo "Cleaning up build context..." + + rm -f \ + "${IMAGE_DIR}/cubelet" \ + "${IMAGE_DIR}/cubecli" \ + "${IMAGE_DIR}/containerd-shim-cube-rs" \ + "${IMAGE_DIR}/cube-runtime" \ + "${IMAGE_DIR}/network-agent" \ + "${IMAGE_DIR}/nicl" \ + "${IMAGE_DIR}/cubelet-code-deploy.sh" \ + "${IMAGE_DIR}/unsquashfs" \ + "${IMAGE_DIR}/unsquashfs-dio" \ + "${IMAGE_DIR}/config-cube.toml" \ + "${IMAGE_DIR}/network-agent.yaml" \ + "${IMAGE_DIR}/snapshot.sh" + + echo " ✓ Cleanup done" + echo "" +} + +# show image info +show_image_info() { + echo "======================================" + echo "Image Information" + echo "======================================" + docker images "${IMAGE_REGISTRY}/cubelet-network-agent" --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.CreatedAt}}" + echo "" + + echo "Usage:" + echo " # Pull image" + echo " docker pull ${IMAGE_REGISTRY}/cubelet-network-agent:${IMAGE_TAG}" + echo "" + echo " # Deploy to Kubernetes (recommended)" + echo " kubectl apply -f deploy/k8s/manifests/namespace.yaml" + echo " kubectl apply -f deploy/k8s/manifests/configmaps/" + echo " kubectl apply -f deploy/k8s/manifests/daemonsets/" + echo "" + echo " # Or run locally for testing:" + echo " docker run -d --name cubelet-test \\" + echo " --privileged \\" + echo " -v /tmp/cube:/tmp/cube \\" + echo " -v /data/cubelet:/data/cubelet \\" + echo " -v /dev/kvm:/dev/kvm \\" + echo " -v /usr/local/services/cubetoolbox/Cubelet/config:/usr/local/services/cubetoolbox/Cubelet/config \\" + echo " -v /usr/local/services/cubetoolbox/Cubelet/dynamicconf:/usr/local/services/cubetoolbox/Cubelet/dynamicconf \\" + echo " ${IMAGE_REGISTRY}/cubelet-network-agent:${IMAGE_TAG}" + echo "" +} + +main() { + check_prebuilt_binaries + prepare_build_context + build_image + push_image + cleanup + show_image_info + + echo "======================================" + echo "Build Completed Successfully!" + echo "======================================" +} + +main "$@" \ No newline at end of file From 817806031b274df8aee7d52d1d903d8479f663e5 Mon Sep 17 00:00:00 2001 From: wwttest <167965921+wwthw@users.noreply.github.com> Date: Fri, 26 Jun 2026 07:54:34 +0000 Subject: [PATCH 2/2] Kubernetes Deployment Configurations for CubeSandbox Control Plane --- .../k8s/manifests/controlplane/cubeapi.yaml | 56 ++++++ .../controlplane/cubemaster-config.yaml | 139 ++++++++++++++ .../controlplane/cubemaster-storage.yaml | 27 +++ .../manifests/controlplane/cubemaster.yaml | 84 +++++++++ .../controlplane/cubeproxy-global-config.yaml | 19 ++ .../k8s/manifests/controlplane/cubeproxy.yaml | 91 +++++++++ .../templates/configmap-coredns.yaml | 68 +++++++ deploy/k8s/manifests/templates/pv.yaml | 42 +++++ deploy/k8s/manifests/templates/pvc.yaml | 42 +++++ deploy/k8s/manifests/templates/secret.yaml | 18 ++ .../manifests/templates/serviceaccount.yaml | 13 ++ deploy/k8s/manifests/templates/services.yaml | 178 ++++++++++++++++++ .../k8s/manifests/templates/statefulset.yaml | 168 +++++++++++++++++ 13 files changed, 945 insertions(+) create mode 100644 deploy/k8s/manifests/controlplane/cubeapi.yaml create mode 100644 deploy/k8s/manifests/controlplane/cubemaster-config.yaml create mode 100644 deploy/k8s/manifests/controlplane/cubemaster-storage.yaml create mode 100644 deploy/k8s/manifests/controlplane/cubemaster.yaml create mode 100644 deploy/k8s/manifests/controlplane/cubeproxy-global-config.yaml create mode 100644 deploy/k8s/manifests/controlplane/cubeproxy.yaml create mode 100644 deploy/k8s/manifests/templates/configmap-coredns.yaml create mode 100644 deploy/k8s/manifests/templates/pv.yaml create mode 100644 deploy/k8s/manifests/templates/pvc.yaml create mode 100644 deploy/k8s/manifests/templates/secret.yaml create mode 100644 deploy/k8s/manifests/templates/serviceaccount.yaml create mode 100644 deploy/k8s/manifests/templates/services.yaml create mode 100644 deploy/k8s/manifests/templates/statefulset.yaml diff --git a/deploy/k8s/manifests/controlplane/cubeapi.yaml b/deploy/k8s/manifests/controlplane/cubeapi.yaml new file mode 100644 index 00000000..d7e93d9a --- /dev/null +++ b/deploy/k8s/manifests/controlplane/cubeapi.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cubeapi + namespace: cube-system + labels: + app: cubeapi + cube-role: control +spec: + replicas: 1 + selector: + matchLabels: + app: cubeapi + template: + metadata: + labels: + app: cubeapi + cube-role: control + spec: + serviceAccountName: cubesandbox + nodeSelector: + cube-role: control + containers: + - name: cubeapi + image: cubeapi:0.3.1 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + name: http + env: + - name: CUBEAPI_BIN + value: "/usr/local/bin/cube-api" + - name: CUBE_MASTER_ADDR + value: "http://cubemaster.cube-system.svc.cluster.local:8089" + command: ["/bin/sh", "-c"] + args: + - | + /usr/local/bin/cube-api --cubemaster-url http://cubemaster.cube-system.svc.cluster.local:8089 & + + wait + volumeMounts: + - name: cubeapi-logs + mountPath: /data/log/CubeAPI + resources: + limits: + cpu: "500m" + memory: "512Mi" + requests: + cpu: "100m" + memory: "128Mi" + volumes: + - name: cubeapi-logs + hostPath: + path: /data/log/CubeAPI + type: DirectoryOrCreate + priorityClassName: system-node-critical \ No newline at end of file diff --git a/deploy/k8s/manifests/controlplane/cubemaster-config.yaml b/deploy/k8s/manifests/controlplane/cubemaster-config.yaml new file mode 100644 index 00000000..858b4eb6 --- /dev/null +++ b/deploy/k8s/manifests/controlplane/cubemaster-config.yaml @@ -0,0 +1,139 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cubemaster-config + namespace: cube-system +data: + conf.yaml: | + common: + http_port: 8089 + http_addr: "0.0.0.0" + http_readtimeout: 120 + http_writetimeout: 360 + http_idletimeout: 360 + sync_meta_data_interval: 30s + sync_metric_data_interval: 1s + collect_metric_interval: 1s + default_headless_service_nodes_num: 1 + enable_check_com_net_id_param: false + + log: + module: "cubemaster" + path: "/data/log/CubeMaster-dev" + file_size: 100 + file_num: 10 + level: "info" + + cubelet_conf: + grpc: + grpc_port: 9999 + common_timeout_insec: 30 + create_image_timeout_insec: 300 + create_concurrent_limit: 100 + destroy_concurent_limit: 100 + enable_exposed_port: true + exposed_port_list: + - "80" + disable_redis_proxy_port: true + + auth: + enable: false + + req_template_conf: + whitelist_req_tag: + WorkingDir: true + RLimit: true + DnsConfig: true + HostAliases: true + Poststop: true + Prestop: true + cube_box_req_template: > + { + "volumes": [ + { + "name": "tmp", + "volume_source": { + "empty_dir": { + "medium": "Memory" + } + } + } + ], + "containers": [ + { + "name": "cubebox-default", + "envs": [ + { + "key": "TZ", + "value": "Asia/Shanghai" + }, + { + "key": "TERM", + "value": "xterm" + } + ] + } + ] + } + + ossdb_config: + addr: "mysql.cube-system.svc.cluster.local:3306" + user: "cube" + pwd: "cube_pass" + db_name: "cube_mvp" + conn_timeout: 5 + read_timeout: 5 + write_timeout: 5 + max_idle_conns: 5 + max_open_conns: 20 + max_conn_life_time_seconds: 300 + + instance_db_config: + addr: "mysql.cube-system.svc.cluster.local:3306" + user: "cube" + pwd: "cube_pass" + db_name: "cube_mvp" + conn_timeout: 5 + read_timeout: 5 + write_timeout: 5 + max_idle_conns: 5 + max_open_conns: 20 + max_conn_life_time_seconds: 300 + + redis: + nodes: "redis.cube-system.svc.cluster.local:6379" + password: "ceuhvu123" + db_no: 0 + max_idle: 8 + max_active: 32 + idle_timeout: 30 + max_retry: 2 + + redis_read: + nodes: "redis.cube-system.svc.cluster.local:6379" + password: "ceuhvu123" + db_no: 0 + max_idle: 8 + max_active: 32 + idle_timeout: 30 + max_retry: 2 + + redis_write: + nodes: "redis.cube-system.svc.cluster.local:6379" + password: "ceuhvu123" + db_no: 0 + max_idle: 8 + max_active: 32 + idle_timeout: 30 + max_retry: 2 + + scheduler: + priority_select_num: 1 + metric_update_timeout: 300s + local_metric_update_timeout: 300s + filter: + enable_filters: + - "cpu" + - "mem" + - "template_locality" + - "realtime_create_num" \ No newline at end of file diff --git a/deploy/k8s/manifests/controlplane/cubemaster-storage.yaml b/deploy/k8s/manifests/controlplane/cubemaster-storage.yaml new file mode 100644 index 00000000..97d7b6d6 --- /dev/null +++ b/deploy/k8s/manifests/controlplane/cubemaster-storage.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: cubemaster-storage-pv-new +spec: + capacity: + storage: 50Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: openclaw-disk-sc + hostPath: + path: /data/cubemaster-storage-new +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cubemaster-storage-pvc-new + namespace: cube-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: openclaw-disk-sc + volumeName: cubemaster-storage-pv-new \ No newline at end of file diff --git a/deploy/k8s/manifests/controlplane/cubemaster.yaml b/deploy/k8s/manifests/controlplane/cubemaster.yaml new file mode 100644 index 00000000..63f0bbc5 --- /dev/null +++ b/deploy/k8s/manifests/controlplane/cubemaster.yaml @@ -0,0 +1,84 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cubemaster + namespace: cube-system + labels: + app: cubemaster + cube-role: control +spec: + replicas: 1 + selector: + matchLabels: + app: cubemaster + template: + metadata: + labels: + app: cubemaster + cube-role: control + spec: + serviceAccountName: cubesandbox + nodeSelector: + cube-role: control + containers: + - name: cubemaster + image: cubemaster:0.3.1 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8089 + name: http + hostPort: 8089 + env: + - name: CUBE_MASTER_CONFIG_PATH + value: "/app/conf/conf.yaml" + volumeMounts: + - name: cubemaster-config + mountPath: /app/conf/conf.yaml + subPath: conf.yaml + - name: cubemaster-logs + mountPath: /data/log/CubeMaster-dev + - name: cubemaster-storage + mountPath: /data/CubeMaster/storage + - name: docker-sock + mountPath: /var/run/docker.sock + command: ["/bin/sh", "-c"] + args: + - | + cubemaster& + wait + resources: + limits: + cpu: "2" + memory: "4Gi" + requests: + cpu: "1" + memory: "2Gi" + volumes: + - name: cubemaster-config + configMap: + name: cubemaster-config + - name: cubemaster-logs + hostPath: + path: /data/log/CubeMaster-dev + - name: cubemaster-storage + persistentVolumeClaim: + claimName: cubemaster-storage-pvc-new + - name: docker-sock + hostPath: + path: /var/run/docker.sock + type: Socket + priorityClassName: system-node-critical +--- +# PVC +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cubemaster-storage-pvc + namespace: cube-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi + storageClassName: standard diff --git a/deploy/k8s/manifests/controlplane/cubeproxy-global-config.yaml b/deploy/k8s/manifests/controlplane/cubeproxy-global-config.yaml new file mode 100644 index 00000000..7b87a1d9 --- /dev/null +++ b/deploy/k8s/manifests/controlplane/cubeproxy-global-config.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cubeproxy-global-conf + namespace: cube-system +data: + global.conf: | + # CubeProxy Redis Configuration + set $redis_ip "redis.cube-system.svc.cluster.local"; + set $redis_port "6379"; + set $redis_pd "ceuhvu123"; + set $redis_index 0; + + # Cache timeout (milliseconds) + set $timeout_min 500; + set $timeout_max 700; + + # Proxy host IP (cubeapi service IP) + set $cube_proxy_host_ip "cubeproxy.cube-system.svc.cluster.local"; \ No newline at end of file diff --git a/deploy/k8s/manifests/controlplane/cubeproxy.yaml b/deploy/k8s/manifests/controlplane/cubeproxy.yaml new file mode 100644 index 00000000..d0893125 --- /dev/null +++ b/deploy/k8s/manifests/controlplane/cubeproxy.yaml @@ -0,0 +1,91 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cubeproxy + namespace: cube-system + labels: + app: cubeproxy + cube-role: control +spec: + replicas: 1 + selector: + matchLabels: + app: cubeproxy + template: + metadata: + labels: + app: cubeproxy + cube-role: control + spec: + #hostNetwork: true + #hostPID: true + serviceAccountName: cubesandbox + nodeSelector: + cube-role: control + + containers: + - name: cubeproxy + image: cubeproxy:0.3.1 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 443 + #hostPort: 443 + name: https-proxy + - containerPort: 80 + #hostPort: 80 + name: http-proxy + env: + - name: CUBE_MASTER_URL + value: 'http://cubemaster:8089' + command: + - /usr/local/openresty/nginx/sbin/start.sh + volumeMounts: + - name: cubeproxy-logs + mountPath: /data/log/cube-proxy + - name: cubeproxy-certs + mountPath: /usr/local/openresty/nginx/certs + readOnly: true + - name: cubeproxy-global-conf + mountPath: /usr/local/openresty/nginx/conf/global/global.conf + subPath: global.conf + resources: + limits: + cpu: "2" + memory: "2Gi" + requests: + cpu: "500m" + memory: "512Mi" + livenessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 3 + readinessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + + volumes: + - name: cubeproxy-logs + hostPath: + path: /data/log/cube-proxy + type: DirectoryOrCreate + - name: cubeproxy-certs + secret: + secretName: cubeproxy-certs + items: + - key: tls.crt + path: cube.app+3.pem + - key: tls.key + path: cube.app+3-key.pem + - name: cubeproxy-global-conf + configMap: + name: cubeproxy-global-conf + items: + - key: global.conf + path: global.conf + + priorityClassName: system-node-critical \ No newline at end of file diff --git a/deploy/k8s/manifests/templates/configmap-coredns.yaml b/deploy/k8s/manifests/templates/configmap-coredns.yaml new file mode 100644 index 00000000..3b68cca7 --- /dev/null +++ b/deploy/k8s/manifests/templates/configmap-coredns.yaml @@ -0,0 +1,68 @@ +--- +# Source: cubesandbox/templates/configmap-coredns.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cubesandbox-coredns-config + namespace: cube-system +data: + Corefile: | + .:53 { + errors + health :5180 + ready + template IN A { + match "^.*\.cube\.app\.$" + answer "{{ .Name }} 60 IN A 10.110.38.229" + } + template IN AAAA { + match "^.*\.cube\.app\.$" + answer "{{ .Name }} 60 IN AAAA ::1" + } + forward . /etc/resolv.conf + log + cache 30 + reload + } +--- +# Source: cubesandbox/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cubesandbox-config + namespace: cube-system +data: + config.yaml: | + common: + http_port: 8089 + http_readtimeout: 120 + http_writetimeout: 360 + http_idletimeout: 360 + log: + module: "cubemaster" + path: "/data/log/cubemaster" + level: "debug" + ossdb_config: + addr: "mysql.cube-system.svc.cluster.local:3306" + user: "cube" + pwd: "cube_pass" + db_name: "cube_mvp" + redis: + nodes: "redis.cube-system.svc.cluster.local:6379" + password: "ceuhvu123" + db_no: 0 + cubelet.yaml: | + common: + common_timeout: "10s" + enable_network_agent: true + network_agent_endpoint: "grpc+unix:///tmp/cube/network-agent-grpc.sock" + host: + scheduler_label: "default-cluster" + global.conf: | + set $redis_ip "redis.cube-system.svc.cluster.local"; + set $redis_port "6379"; + set $redis_pd "ceuhvu123"; + set $redis_index 0; + set $cube_proxy_host_ip "10.110.8.57"; + set $timeout_min 500; + set $timeout_max 700; \ No newline at end of file diff --git a/deploy/k8s/manifests/templates/pv.yaml b/deploy/k8s/manifests/templates/pv.yaml new file mode 100644 index 00000000..16a88b63 --- /dev/null +++ b/deploy/k8s/manifests/templates/pv.yaml @@ -0,0 +1,42 @@ +--- +# Source: cubesandbox/templates/pv.yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: cubesandbox-mysql-pv + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: mysql +spec: + capacity: + storage: 100Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /mnt/data/mysql +--- +# Source: cubesandbox/templates/pv.yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: cubesandbox-redis-pv + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: redis +spec: + capacity: + storage: 50Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /mnt/data/redis \ No newline at end of file diff --git a/deploy/k8s/manifests/templates/pvc.yaml b/deploy/k8s/manifests/templates/pvc.yaml new file mode 100644 index 00000000..a8e67196 --- /dev/null +++ b/deploy/k8s/manifests/templates/pvc.yaml @@ -0,0 +1,42 @@ +--- +# Source: cubesandbox/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cubesandbox-mysql + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: mysql +spec: + accessModes: + - ReadWriteOnce + volumeName: cubesandbox-mysql-pv + resources: + requests: + storage: 100Gi +--- +# Source: cubesandbox/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cubesandbox-redis + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: redis +spec: + accessModes: + - ReadWriteOnce + volumeName: cubesandbox-redis-pv + resources: + requests: + storage: 50Gi \ No newline at end of file diff --git a/deploy/k8s/manifests/templates/secret.yaml b/deploy/k8s/manifests/templates/secret.yaml new file mode 100644 index 00000000..91cff2c3 --- /dev/null +++ b/deploy/k8s/manifests/templates/secret.yaml @@ -0,0 +1,18 @@ +--- +# Source: cubesandbox/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: cubesandbox-secret + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm +type: Opaque +stringData: + mysql-root-password: "root_pass" + mysql-password: "cube_pass" + redis-password: "ceuhvu123" \ No newline at end of file diff --git a/deploy/k8s/manifests/templates/serviceaccount.yaml b/deploy/k8s/manifests/templates/serviceaccount.yaml new file mode 100644 index 00000000..c23b740f --- /dev/null +++ b/deploy/k8s/manifests/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +--- +# Source: cubesandbox/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cubesandbox + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm \ No newline at end of file diff --git a/deploy/k8s/manifests/templates/services.yaml b/deploy/k8s/manifests/templates/services.yaml new file mode 100644 index 00000000..2a036340 --- /dev/null +++ b/deploy/k8s/manifests/templates/services.yaml @@ -0,0 +1,178 @@ +--- +# Source: cubesandbox/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: mysql +spec: + type: ClusterIP + ports: + - port: 3306 + targetPort: mysql + protocol: TCP + name: mysql + selector: + component: mysql +--- +# Source: cubesandbox/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: redis +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: redis + protocol: TCP + name: redis + selector: + component: redis +--- +# Source: cubesandbox/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: cubeapi + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: cubeapi +spec: + type: ClusterIP + ports: + - port: 8088 + targetPort: 8088 + protocol: TCP + name: http + selector: + component: cubeapi +--- +# Source: cubesandbox/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: cubemaster + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: cubemaster +spec: + type: ClusterIP + ports: + - port: 8089 + targetPort: 8089 + protocol: TCP + name: http + - port: 9999 + targetPort: 9999 + protocol: TCP + name: grpc + selector: + component: cubemaster +--- +# Source: cubesandbox/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: cubeproxy + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: cubeproxy +spec: + type: NodePort + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http + - port: 443 + targetPort: 443 + protocol: TCP + name: https + - port: 8081 + targetPort: 8081 + protocol: TCP + name: admin + selector: + component: cubeproxy +--- +# Source: cubesandbox/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: cubedns + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: coredns +spec: + type: ClusterIP + ports: + - port: 53 + targetPort: 53 + protocol: UDP + name: dns-udp + - port: 53 + targetPort: 53 + protocol: TCP + name: dns-tcp + selector: + component: coredns +--- +# Source: cubesandbox/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: cubenode + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: cubenode +spec: + type: ClusterIP + clusterIP: None + ports: + - port: 9999 + targetPort: 9999 + protocol: TCP + name: grpc + selector: + component: cubenode \ No newline at end of file diff --git a/deploy/k8s/manifests/templates/statefulset.yaml b/deploy/k8s/manifests/templates/statefulset.yaml new file mode 100644 index 00000000..191ac628 --- /dev/null +++ b/deploy/k8s/manifests/templates/statefulset.yaml @@ -0,0 +1,168 @@ +--- +# Source: cubesandbox/templates/statefulset-mysql.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mysql + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: mysql +spec: + serviceName: mysql + replicas: 1 + selector: + matchLabels: + component: mysql + template: + metadata: + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: mysql + spec: + serviceAccountName: cubesandbox + containers: + - name: mysql + image: "mysql:8.0" + imagePullPolicy: IfNotPresent + ports: + - name: mysql + containerPort: 3306 + protocol: TCP + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: cubesandbox-secret + key: mysql-root-password + - name: MYSQL_DATABASE + value: cube_mvp + - name: MYSQL_USER + value: cube + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: cubesandbox-secret + key: mysql-password + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + resources: + limits: + cpu: 2000m + memory: 4Gi + requests: + cpu: 500m + memory: 1Gi + livenessProbe: + exec: + command: + - mysqladmin + - ping + - -h + - localhost + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - "mysqladmin ping -h localhost" + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 10 + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: cubesandbox-mysql +--- +# Source: cubesandbox/templates/statefulset-redis.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: cube-system + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: redis +spec: + serviceName: redis + replicas: 1 + selector: + matchLabels: + component: redis + template: + metadata: + labels: + helm.sh/chart: cubesandbox-1.0.0 + app.kubernetes.io/name: cubesandbox + app.kubernetes.io/instance: cubesandbox + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm + component: redis + spec: + serviceAccountName: cubesandbox + containers: + - name: redis + image: "redis:7.2-alpine" + imagePullPolicy: IfNotPresent + ports: + - name: redis + containerPort: 6379 + protocol: TCP + command: + - redis-server + - --requirepass + - "$(REDIS_PASSWORD)" + - --appendonly + - "yes" + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: cubesandbox-secret + key: redis-password + volumeMounts: + - name: redis-data + mountPath: /data + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 250m + memory: 512Mi + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 10 + volumes: + - name: redis-data + persistentVolumeClaim: + claimName: cubesandbox-redis \ No newline at end of file