diff --git a/.gitignore b/.gitignore index 6603f927d..ee7e248e6 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,9 @@ dist .ruff_cache .env .tox + +# Ignore charts directory +simplyblock_core/scripts/charts/charts/ + +# Ignore Helm requirements lock file +simplyblock_core/scripts/charts/requirements.lock diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 5e6352cc0..fa11d2f4c 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -80,7 +80,7 @@ def _create_update_user(cluster_id, grafana_url, grafana_secret, user_secret, up def _add_graylog_input(cluster_ip, password): - base_url = f"http://{cluster_ip}/graylog/api" + base_url = f"{cluster_ip}/api" input_url = f"{base_url}/system/inputs" retries = 30 @@ -161,7 +161,7 @@ def _add_graylog_input(cluster_ip, password): def _set_max_result_window(cluster_ip, max_window=100000): - url_existing_indices = f"http://{cluster_ip}/opensearch/_all/_settings" + url_existing_indices = f"{cluster_ip}/_all/_settings" retries = 30 reachable=False @@ -188,7 +188,7 @@ def _set_max_result_window(cluster_ip, max_window=100000): logger.error(f"Failed to update settings for existing indices: {response.text}") return False - url_template = f"http://{cluster_ip}/opensearch/_template/all_indices_template" + url_template = f"{cluster_ip}/_template/all_indices_template" payload_template = json.dumps({ "index_patterns": ["*"], "settings": { @@ -317,8 +317,12 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.grafana_endpoint = grafana_endpoint elif ingress_host_source == "hostip": cluster.grafana_endpoint = f"http://{dev_ip}/grafana" + graylog_endpoint = f"http://{dev_ip}/graylog" + os_endpoint = f"http://{dev_ip}/opensearch" else: cluster.grafana_endpoint = f"http://{dns_name}/grafana" + graylog_endpoint = f"http://{dns_name}/graylog" + os_endpoint = f"http://{dns_name}/opensearch" cluster.enable_node_affinity = enable_node_affinity cluster.qpair_count = qpair_count or constants.QPAIR_COUNT cluster.client_qpair_count = client_qpair_count or constants.CLIENT_QPAIR_COUNT @@ -355,9 +359,10 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, if ingress_host_source == "hostip": dns_name = dev_ip - _set_max_result_window(dns_name) + + _set_max_result_window(os_endpoint) - _add_graylog_input(dns_name, monitoring_secret) + _add_graylog_input(graylog_endpoint, monitoring_secret) _create_update_user(cluster.uuid, cluster.grafana_endpoint, monitoring_secret, cluster.secret) if mode == "kubernetes": @@ -433,12 +438,13 @@ def _run_fio(mount_point) -> None: def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, enable_node_affinity, qpair_count, - max_queue_size, inflight_io_threshold, strict_node_anti_affinity, is_single_node, name, fabric="tcp", - cluster_ip=None, grafana_secret=None) -> str: + max_queue_size, inflight_io_threshold, strict_node_anti_affinity, is_single_node, name, cr_name=None, + cr_namespace=None, cr_plural=None, fabric="tcp", cluster_ip=None, grafana_secret=None) -> str: default_cluster = None monitoring_secret = os.environ.get("MONITORING_SECRET", "") + enable_monitoring = os.environ.get("ENABLE_MONITORING", "") clusters = db_controller.get_clusters() if clusters: default_cluster = clusters[0] @@ -471,16 +477,27 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn cluster.db_connection = fdb_cluster_string if monitoring_secret: cluster.grafana_secret = monitoring_secret + elif enable_monitoring != "true": + cluster.grafana_secret = "" else: raise Exception("monitoring_secret is required") - cluster.grafana_endpoint = "http://simplyblock-grafana:3000" + cluster.grafana_endpoint = constants.GRAFANA_K8S_ENDPOINT if not cluster_ip: cluster_ip = "0.0.0.0" # add mgmt node object mgmt_node_ops.add_mgmt_node(cluster_ip, "kubernetes", cluster.uuid) - - _create_update_user(cluster.uuid, cluster.grafana_endpoint, cluster.grafana_secret, cluster.secret) + if enable_monitoring == "true": + graylog_endpoint = constants.GRAYLOG_K8S_ENDPOINT + os_endpoint = constants.OS_K8S_ENDPOINT + _create_update_user(cluster.uuid, cluster.grafana_endpoint, cluster.grafana_secret, cluster.secret) + + _set_max_result_window(os_endpoint) + + _add_graylog_input(graylog_endpoint, monitoring_secret) + + if cluster.mode == "kubernetes": + utils.patch_prometheus_configmap(cluster.uuid, cluster.secret) cluster.distr_ndcs = distr_ndcs cluster.distr_npcs = distr_npcs @@ -492,6 +509,10 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn cluster.qpair_count = qpair_count or constants.QPAIR_COUNT cluster.max_queue_size = max_queue_size cluster.inflight_io_threshold = inflight_io_threshold + cluster.cr_name = cr_name + cluster.cr_namespace = cr_namespace + cluster.cr_plural = cr_plural + if cap_warn and cap_warn > 0: cluster.cap_warn = cap_warn if cap_crit and cap_crit > 0: diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 36ba14a9e..08b101d0c 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -167,6 +167,15 @@ def get_config_var(name, default=None): LVO_MAX_NAMESPACES_PER_SUBSYS=32 +CR_GROUP = "simplyblock.simplyblock.io" +CR_VERSION = "v1alpha1" + +GRAFANA_K8S_ENDPOINT = "http://simplyblock-grafana:3000" +GRAYLOG_K8S_ENDPOINT = "http://simplyblock-graylog:9000" +OS_K8S_ENDPOINT = "http://opensearch-cluster-master:9200" + +WEBAPI_K8S_ENDPOINT = "http://simplyblock-webappapi:5000/api/v2" + K8S_NAMESPACE = os.getenv('K8S_NAMESPACE', 'simplyblock') OS_STATEFULSET_NAME = "simplyblock-opensearch" MONGODB_STATEFULSET_NAME = "simplyblock-mongo" diff --git a/simplyblock_core/controllers/cluster_events.py b/simplyblock_core/controllers/cluster_events.py index 059aea976..e201c53a9 100644 --- a/simplyblock_core/controllers/cluster_events.py +++ b/simplyblock_core/controllers/cluster_events.py @@ -4,6 +4,7 @@ from simplyblock_core.controllers import events_controller as ec from simplyblock_core.db_controller import DBController from simplyblock_core.models.events import EventObj +from simplyblock_core import utils, constants logger = logging.getLogger() db_controller = DBController() @@ -39,6 +40,15 @@ def cluster_status_change(cluster, new_state, old_status): caused_by=ec.CAUSED_BY_CLI, message=f"Cluster status changed from {old_status} to {new_state}") + if cluster.mode == "kubernetes": + utils.patch_cr_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=cluster.cr_plural, + namespace=cluster.cr_namespace, + name=cluster.cr_name, + status_patch={"status": new_state}) + def _cluster_cap_event(cluster, msg, event_level): return ec.log_event_cluster( @@ -90,3 +100,11 @@ def cluster_rebalancing_change(cluster, new_state, old_status): db_object=cluster, caused_by=ec.CAUSED_BY_CLI, message=f"Cluster rebalancing changed from {old_status} to {new_state}") + if cluster.mode == "kubernetes": + utils.patch_cr_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=cluster.cr_plural, + namespace=cluster.cr_namespace, + name=cluster.cr_name, + status_patch={"rebalancing": new_state}) diff --git a/simplyblock_core/controllers/device_events.py b/simplyblock_core/controllers/device_events.py index f2e1e959d..1f5ee881a 100644 --- a/simplyblock_core/controllers/device_events.py +++ b/simplyblock_core/controllers/device_events.py @@ -3,6 +3,8 @@ from simplyblock_core.controllers import events_controller as ec from simplyblock_core.db_controller import DBController +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core import utils, constants logger = logging.getLogger() @@ -20,6 +22,24 @@ def _device_event(device, message, caused_by, event): node_id=device.get_id(), storage_id=device.cluster_device_order) + cluster = db_controller.get_cluster_by_id(snode.cluster_id) + if cluster.mode == "kubernetes": + total_devices = len(snode.nvme_devices) + online_devices = 0 + for dev in snode.nvme_devices: + if dev.status == NVMeDevice.STATUS_ONLINE: + online_devices += 1 + utils.patch_cr_node_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=snode.cr_plural, + namespace=snode.cr_namespace, + name=snode.cr_name, + node_uuid=snode.get_id(), + node_mgmt_ip=snode.mgmt_ip, + updates={"devices": f"{total_devices}/{online_devices}"}, + ) + def device_create(device, caused_by=ec.CAUSED_BY_CLI): _device_event(device, f"Device created: {device.get_id()}", caused_by, ec.EVENT_OBJ_CREATED) diff --git a/simplyblock_core/controllers/lvol_events.py b/simplyblock_core/controllers/lvol_events.py index 636c444b3..91b91027b 100644 --- a/simplyblock_core/controllers/lvol_events.py +++ b/simplyblock_core/controllers/lvol_events.py @@ -3,6 +3,7 @@ from simplyblock_core.controllers import events_controller as ec from simplyblock_core.db_controller import DBController +from simplyblock_core import utils, constants logger = logging.getLogger() @@ -10,6 +11,7 @@ def _lvol_event(lvol, message, caused_by, event): db_controller = DBController() snode = db_controller.get_storage_node_by_id(lvol.node_id) + cluster = db_controller.get_cluster_by_id(snode.cluster_id) ec.log_event_cluster( cluster_id=snode.cluster_id, domain=ec.DOMAIN_CLUSTER, @@ -18,7 +20,79 @@ def _lvol_event(lvol, message, caused_by, event): caused_by=caused_by, message=message, node_id=lvol.get_id()) - + if cluster.mode == "kubernetes": + pool = db_controller.get_pool_by_id(lvol.pool_uuid) + + if event == ec.EVENT_OBJ_CREATED: + crypto_key=( + (lvol.crypto_key1, lvol.crypto_key2) + if lvol.crypto_key1 and lvol.crypto_key2 + else None + ) + + node_urls = [ + f"{constants.WEBAPI_K8S_ENDPOINT}/clusters/{snode.cluster_id}/storage-nodes/{node_id}/" + for node_id in lvol.nodes + ] + + utils.patch_cr_lvol_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=pool.lvols_cr_plural, + namespace=pool.lvols_cr_namespace, + name=pool.lvols_cr_name, + add={ + "uuid": lvol.get_id(), + "lvolName": lvol.lvol_name, + "status": lvol.status, + "nodeUUID": node_urls, + "size": utils.humanbytes(lvol.size), + "health": lvol.health_check, + "isCrypto": crypto_key is not None, + "nqn": lvol.nqn, + "subsysPort": lvol.subsys_port, + "hostname": lvol.hostname, + "fabric": lvol.fabric, + "ha": lvol.ha_type == 'ha', + "poolUUID": lvol.pool_uuid, + "poolName": lvol.pool_name, + "PvcName": lvol.pvc_name, + "snapName": lvol.snapshot_name, + "clonedFromSnap": lvol.cloned_from_snap, + "stripeWdata": lvol.ndcs, + "stripeWparity": lvol.npcs, + "blobID": lvol.blobid, + "namespaceID": lvol.ns_id, + "qosClass": lvol.lvol_priority_class, + "maxNamespacesPerSubsystem": lvol.max_namespace_per_subsys, + "qosIOPS": lvol.rw_ios_per_sec, + "qosRWTP": lvol.rw_mbytes_per_sec, + "qosRTP": lvol.r_mbytes_per_sec, + "qosWTP": lvol.w_mbytes_per_sec, + }, + ) + + elif event == ec.EVENT_STATUS_CHANGE: + utils.patch_cr_lvol_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=pool.lvols_cr_plural, + namespace=pool.lvols_cr_namespace, + name=pool.lvols_cr_name, + lvol_uuid=lvol.get_id(), + updates={"status": lvol.status, "health": lvol.health_check}, + ) + elif event == ec.EVENT_OBJ_DELETED: + logger.info("Deleting lvol CR object") + utils.patch_cr_lvol_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=pool.lvols_cr_plural, + namespace=pool.lvols_cr_namespace, + name=pool.lvols_cr_name, + lvol_uuid=lvol.get_id(), + remove=True, + ) def lvol_create(lvol, caused_by=ec.CAUSED_BY_CLI): _lvol_event(lvol, "LVol created", caused_by, ec.EVENT_OBJ_CREATED) diff --git a/simplyblock_core/controllers/pool_controller.py b/simplyblock_core/controllers/pool_controller.py index 2440a6bd7..0d2738e67 100644 --- a/simplyblock_core/controllers/pool_controller.py +++ b/simplyblock_core/controllers/pool_controller.py @@ -23,7 +23,8 @@ def _generate_string(length): string.ascii_letters + string.digits) for _ in range(length)) -def add_pool(name, pool_max, lvol_max, max_rw_iops, max_rw_mbytes, max_r_mbytes, max_w_mbytes, cluster_id, qos_host=None): +def add_pool(name, pool_max, lvol_max, max_rw_iops, max_rw_mbytes, max_r_mbytes, max_w_mbytes, cluster_id, + cr_name=None, cr_namespace=None, cr_plural=None, qos_host=None): db_controller = DBController() if not name: logger.error("Pool name is empty!") @@ -71,6 +72,9 @@ def add_pool(name, pool_max, lvol_max, max_rw_iops, max_rw_mbytes, max_r_mbytes, pool.max_rw_mbytes_per_sec = max_rw_mbytes pool.max_r_mbytes_per_sec = max_r_mbytes pool.max_w_mbytes_per_sec = max_w_mbytes + pool.cr_name = cr_name + pool.cr_namespace = cr_namespace + pool.cr_plural = cr_plural if pool.has_qos() and not qos_host: next_nodes = lvol_controller._get_next_3_nodes(cluster_id) if next_nodes: @@ -121,7 +125,8 @@ def qos_exists_on_child_lvol(db_controller: DBController, pool_uuid): return False def set_pool(uuid, pool_max=0, lvol_max=0, max_rw_iops=0, - max_rw_mbytes=0, max_r_mbytes=0, max_w_mbytes=0, name=""): + max_rw_mbytes=0, max_r_mbytes=0, max_w_mbytes=0, name="", + lvols_cr_name="", lvols_cr_namespace="", lvols_cr_plural=""): db_controller = DBController() try: pool = db_controller.get_pool_by_id(uuid) @@ -143,6 +148,17 @@ def set_pool(uuid, pool_max=0, lvol_max=0, max_rw_iops=0, return False, msg pool.pool_name = name + if lvols_cr_name and lvols_cr_name != pool.lvols_cr_name: + for p in db_controller.get_pools(): + if p.lvols_cr_name == lvols_cr_name: + msg = f"Pool found with the same lvol cr name: {name}" + logger.error(msg) + return False, msg + pool.lvols_cr_name = lvols_cr_name + pool.lvols_cr_namespace = lvols_cr_namespace + pool.lvols_cr_plural = lvols_cr_plural + + # Normalize inputs max_rw_iops = max_rw_iops or 0 max_rw_mbytes = max_rw_mbytes or 0 @@ -265,8 +281,10 @@ def set_status(pool_id, status): except KeyError: logger.error(f"Pool not found {pool_id}") return False + old_status = pool.status pool.status = status pool.write_to_db(db_controller.kv_store) + pool_events.pool_status_change(pool, pool.status, old_status) logger.info("Done") diff --git a/simplyblock_core/controllers/pool_events.py b/simplyblock_core/controllers/pool_events.py index 2581d59b1..8c4f0ea08 100644 --- a/simplyblock_core/controllers/pool_events.py +++ b/simplyblock_core/controllers/pool_events.py @@ -2,7 +2,8 @@ import logging from simplyblock_core.controllers import events_controller as ec - +from simplyblock_core.db_controller import DBController +from simplyblock_core import utils, constants logger = logging.getLogger() @@ -29,3 +30,24 @@ def pool_remove(pool): def pool_updated(pool): _add(pool, f"Pool updated {pool.pool_name}", event=ec.EVENT_STATUS_CHANGE) + +def pool_status_change(pool, new_state, old_status): + db_controller = DBController() + cluster = db_controller.get_cluster_by_id(pool.cluster_id) + ec.log_event_cluster( + cluster_id=pool.cluster_id, + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_STATUS_CHANGE, + db_object=pool, + caused_by=ec.CAUSED_BY_CLI, + message=f"Pool status changed from {old_status} to {new_state}", + node_id=pool.cluster_id) + + if cluster.mode == "kubernetes": + utils.patch_cr_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=pool.cr_plural, + namespace=pool.cr_namespace, + name=pool.cr_name, + status_patch={"status": new_state}) diff --git a/simplyblock_core/controllers/storage_events.py b/simplyblock_core/controllers/storage_events.py index 027f7dbed..6bef257fd 100644 --- a/simplyblock_core/controllers/storage_events.py +++ b/simplyblock_core/controllers/storage_events.py @@ -3,6 +3,8 @@ from simplyblock_core.controllers import events_controller as ec from simplyblock_core.models.events import EventObj +from simplyblock_core.db_controller import DBController +from simplyblock_core import utils, constants logger = logging.getLogger() @@ -19,6 +21,8 @@ def snode_add(node): def snode_delete(node): + db_controller = DBController() + cluster = db_controller.get_cluster_by_id(node.cluster_id) ec.log_event_cluster( cluster_id=node.cluster_id, domain=ec.DOMAIN_CLUSTER, @@ -27,9 +31,21 @@ def snode_delete(node): caused_by=ec.CAUSED_BY_CLI, message=f"Storage node deleted {node.get_id()}", node_id=node.get_id()) - + if cluster.mode == "kubernetes": + utils.patch_cr_node_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=node.cr_plural, + namespace=node.cr_namespace, + name=node.cr_name, + node_uuid=node.get_id(), + node_mgmt_ip=node.mgmt_ip, + remove=True, + ) def snode_status_change(node, new_state, old_status, caused_by=ec.CAUSED_BY_CLI): + db_controller = DBController() + cluster = db_controller.get_cluster_by_id(node.cluster_id) ec.log_event_cluster( cluster_id=node.cluster_id, domain=ec.DOMAIN_CLUSTER, @@ -38,9 +54,22 @@ def snode_status_change(node, new_state, old_status, caused_by=ec.CAUSED_BY_CLI) caused_by=caused_by, message=f"Storage node status changed from: {old_status} to: {new_state}", node_id=node.get_id()) + if cluster.mode == "kubernetes": + utils.patch_cr_node_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=node.cr_plural, + namespace=node.cr_namespace, + name=node.cr_name, + node_uuid=node.get_id(), + node_mgmt_ip=node.mgmt_ip, + updates={"status": new_state}, + ) def snode_health_check_change(node, new_state, old_status, caused_by=ec.CAUSED_BY_CLI): + db_controller = DBController() + cluster = db_controller.get_cluster_by_id(node.cluster_id) ec.log_event_cluster( cluster_id=node.cluster_id, domain=ec.DOMAIN_CLUSTER, @@ -49,7 +78,17 @@ def snode_health_check_change(node, new_state, old_status, caused_by=ec.CAUSED_B caused_by=caused_by, message=f"Storage node health check changed from: {old_status} to: {new_state}", node_id=node.get_id()) - + if cluster.mode == "kubernetes": + utils.patch_cr_node_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=node.cr_plural, + namespace=node.cr_namespace, + name=node.cr_name, + node_uuid=node.get_id(), + node_mgmt_ip=node.mgmt_ip, + updates={"health": new_state}, + ) def snode_restart_failed(node): ec.log_event_cluster( @@ -87,6 +126,8 @@ def jm_repl_tasks_found(node, jm_vuid, caused_by=ec.CAUSED_BY_MONITOR): def node_ports_changed(node, caused_by=ec.CAUSED_BY_MONITOR): + db_controller = DBController() + cluster = db_controller.get_cluster_by_id(node.cluster_id) ec.log_event_cluster( cluster_id=node.cluster_id, domain=ec.DOMAIN_CLUSTER, @@ -96,3 +137,15 @@ def node_ports_changed(node, caused_by=ec.CAUSED_BY_MONITOR): event_level=EventObj.LEVEL_WARN, message=f"Storage node ports set, LVol:{node.lvol_subsys_port} RPC:{node.rpc_port} Internal:{node.nvmf_port}", node_id=node.get_id()) + if cluster.mode == "kubernetes": + utils.patch_cr_node_status( + group=constants.CR_GROUP, + version=constants.CR_VERSION, + plural=node.cr_plural, + namespace=node.cr_namespace, + name=node.cr_name, + node_uuid=node.get_id(), + node_mgmt_ip=node.mgmt_ip, + updates={"nvmf_port": node.nvmf_port, "rpc_port": node.rpc_port, "lvol_port": node.lvol_subsys_port}, + ) + \ No newline at end of file diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 620309f77..f85be6e06 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -63,6 +63,9 @@ class Cluster(BaseModel): fabric_rdma: bool = False client_qpair_count: int = 3 secret: str = "" + cr_name: str = "" + cr_namespace: str = "" + cr_plural: str = "" disable_monitoring: bool = False strict_node_anti_affinity: bool = False tls: bool = False diff --git a/simplyblock_core/models/pool.py b/simplyblock_core/models/pool.py index 27b2a23e5..683eafe1e 100644 --- a/simplyblock_core/models/pool.py +++ b/simplyblock_core/models/pool.py @@ -29,6 +29,12 @@ class Pool(BaseModel): secret: str = "" # unused users: List[str] = [] qos_host: str = "" + cr_name: str = "" + cr_namespace: str = "" + cr_plural: str = "" + lvols_cr_name: str = "" + lvols_cr_namespace: str = "" + lvols_cr_plural: str = "" def has_qos(self): diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 45abceec9..ab7b31b09 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -97,6 +97,9 @@ class StorageNode(BaseNodeObject): subsystem: str = "" system_uuid: str = "" lvstore_status: str = "" + cr_name: str = "" + cr_namespace: str = "" + cr_plural: str = "" nvmf_port: int = 4420 physical_label: int = 0 hublvol: HubLVol = None # type: ignore[assignment] diff --git a/simplyblock_core/scripts/charts/Chart.yaml b/simplyblock_core/scripts/charts/Chart.yaml index 380f67bcd..671f39cfa 100644 --- a/simplyblock_core/scripts/charts/Chart.yaml +++ b/simplyblock_core/scripts/charts/Chart.yaml @@ -17,15 +17,14 @@ dependencies: version: 1.4.0 repository: https://mongodb.github.io/helm-charts alias: mongodb - condition: monitoring.enabled + condition: observability.enabled - name: opensearch version: 2.9.0 repository: https://opensearch-project.github.io/helm-charts - condition: monitoring.enabled + condition: observability.enabled - name: prometheus version: "25.18.0" repository: "https://prometheus-community.github.io/helm-charts" - condition: monitoring.enabled - name: ingress-nginx version: 4.10.1 repository: "https://kubernetes.github.io/ingress-nginx" diff --git a/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockdevices.yaml b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockdevices.yaml new file mode 100644 index 000000000..272030736 --- /dev/null +++ b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockdevices.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: simplyblockdevices.simplyblock.simplyblock.io +spec: + group: simplyblock.simplyblock.io + names: + kind: SimplyBlockDevice + listKind: SimplyBlockDeviceList + plural: simplyblockdevices + singular: simplyblockdevice + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SimplyBlockDevice is the Schema for the simplyblockdevices API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SimplyBlockDevice + properties: + action: + enum: + - remove + - restart + type: string + clusterName: + type: string + deviceID: + type: string + nodeUUID: + type: string + required: + - clusterName + type: object + status: + description: status defines the observed state of SimplyBlockDevice + properties: + actionStatus: + properties: + action: + type: string + message: + type: string + nodeUUID: + type: string + observedGeneration: + format: int64 + type: integer + state: + type: string + triggered: + type: boolean + updatedAt: + format: date-time + type: string + type: object + nodes: + items: + properties: + devices: + items: + properties: + health: + type: string + model: + type: string + size: + type: string + stats: + items: + properties: + capacityUtil: + format: int64 + type: integer + riops: + format: int64 + type: integer + rtp: + format: int64 + type: integer + wiops: + format: int64 + type: integer + wtp: + format: int64 + type: integer + type: object + type: array + status: + type: string + utilization: + format: int64 + type: integer + uuid: + type: string + type: object + type: array + nodeUUID: + type: string + type: object + type: array + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: nodeUUID and deviceID are required when action is specified + rule: '!(has(self.spec.action) && self.spec.action != "" && ((!has(self.spec.nodeUUID) + || self.spec.nodeUUID == "") || (!has(self.spec.deviceID) || self.spec.deviceID + == "")))' + served: true + storage: true + subresources: + status: {} diff --git a/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblocklvols.yaml b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblocklvols.yaml new file mode 100644 index 000000000..8e44a687d --- /dev/null +++ b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblocklvols.yaml @@ -0,0 +1,144 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: simplyblocklvols.simplyblock.simplyblock.io +spec: + group: simplyblock.simplyblock.io + names: + kind: SimplyBlockLvol + listKind: SimplyBlockLvolList + plural: simplyblocklvols + singular: simplyblocklvol + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.lvols.length() + name: LVOLs + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: SimplyBlockLvol is the Schema for the simplyblocklvols API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SimplyBlockLvol + properties: + clusterName: + type: string + poolName: + type: string + required: + - clusterName + - poolName + type: object + status: + description: status defines the observed state of SimplyBlockLvol + properties: + configured: + type: boolean + lvols: + items: + properties: + blobID: + format: int64 + type: integer + clonedFromSnap: + type: string + createDt: + format: date-time + type: string + fabric: + type: string + ha: + type: boolean + health: + type: boolean + hostname: + type: string + isCrypto: + type: boolean + lvolName: + type: string + maxNamespacesPerSubsystem: + format: int64 + type: integer + namespaceID: + format: int64 + type: integer + nodeUUID: + items: + type: string + type: array + nqn: + type: string + poolName: + type: string + poolUUID: + type: string + pvcName: + type: string + qosClass: + format: int64 + type: integer + qosIOPS: + format: int64 + type: integer + qosRTP: + format: int64 + type: integer + qosRWTP: + format: int64 + type: integer + qosWTP: + format: int64 + type: integer + size: + type: string + snapName: + type: string + status: + type: string + stripeWdata: + format: int64 + type: integer + stripeWparity: + format: int64 + type: integer + subsysPort: + format: int64 + type: integer + updateDt: + format: date-time + type: string + uuid: + type: string + type: object + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockpools.yaml b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockpools.yaml new file mode 100644 index 000000000..693322dc3 --- /dev/null +++ b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockpools.yaml @@ -0,0 +1,96 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: simplyblockpools.simplyblock.simplyblock.io +spec: + group: simplyblock.simplyblock.io + names: + kind: SimplyBlockPool + listKind: SimplyBlockPoolList + plural: simplyblockpools + singular: simplyblockpool + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SimplyBlockPool is the Schema for the pools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of Pool + properties: + action: + type: string + capacityLimit: + type: string + clusterName: + type: string + name: + type: string + qosIOPSLimit: + format: int32 + type: integer + rLimit: + format: int32 + type: integer + rwLimit: + format: int32 + type: integer + status: + type: string + wLimit: + format: int32 + type: integer + required: + - clusterName + - name + type: object + status: + description: status defines the observed state of Pool + properties: + qosHost: + type: string + qosIOPSLimit: + format: int32 + type: integer + rLimit: + format: int32 + type: integer + rwLimit: + format: int32 + type: integer + status: + type: string + uuid: + type: string + wLimit: + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockstorageclusters.yaml b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockstorageclusters.yaml new file mode 100644 index 000000000..cfd99fdee --- /dev/null +++ b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockstorageclusters.yaml @@ -0,0 +1,173 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: simplyblockstorageclusters.simplyblock.simplyblock.io +spec: + group: simplyblock.simplyblock.io + names: + kind: SimplyBlockStorageCluster + listKind: SimplyBlockStorageClusterList + plural: simplyblockstorageclusters + singular: simplyblockstoragecluster + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SimplyBlockStorageCluster is the Schema for the simplyblockstorageclusters + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SimplyBlockStorageCluster + properties: + action: + enum: + - activate + - expand + type: string + blkSize: + format: int32 + type: integer + capCrit: + format: int32 + type: integer + capWarn: + format: int32 + type: integer + clientQpairCount: + format: int32 + type: integer + clusterName: + type: string + distrBs: + format: int32 + type: integer + distrChunkBs: + format: int32 + type: integer + enableNodeAffinity: + type: boolean + eventLogEntries: + format: int32 + type: integer + fabric: + type: string + haType: + type: string + includeEventLog: + type: boolean + inflightIOThreshold: + format: int32 + type: integer + isSingleNode: + type: boolean + maxQueueSize: + format: int32 + type: integer + mgmtIfc: + description: Create-only + type: string + pageSizeInBlocks: + format: int32 + type: integer + provCapCrit: + format: int32 + type: integer + provCapWarn: + format: int32 + type: integer + qosClasses: + description: Updatable + type: string + qpairCount: + format: int32 + type: integer + strictNodeAntiAffinity: + type: boolean + stripeWdata: + format: int32 + type: integer + stripeWparity: + format: int32 + type: integer + required: + - clusterName + type: object + status: + description: status defines the observed state of SimplyBlockStorageCluster + properties: + MOD: + type: string + NQN: + type: string + UUID: + type: string + actionStatus: + properties: + action: + type: string + message: + type: string + nodeUUID: + type: string + observedGeneration: + format: int64 + type: integer + state: + type: string + triggered: + type: boolean + updatedAt: + format: date-time + type: string + type: object + clusterName: + type: string + configured: + type: boolean + created: + format: date-time + type: string + lastUpdated: + format: date-time + type: string + mgmtNodes: + format: int32 + type: integer + rebalancing: + type: boolean + secretName: + type: string + status: + type: string + storageNodes: + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockstoragenodes.yaml b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockstoragenodes.yaml new file mode 100644 index 000000000..1e6af7724 --- /dev/null +++ b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblockstoragenodes.yaml @@ -0,0 +1,204 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: simplyblockstoragenodes.simplyblock.simplyblock.io +spec: + group: simplyblock.simplyblock.io + names: + kind: SimplyBlockStorageNode + listKind: SimplyBlockStorageNodeList + plural: simplyblockstoragenodes + singular: simplyblockstoragenode + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SimplyBlockStorageNode is the Schema for the storagenodes API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of StorageNode + properties: + action: + enum: + - shutdown + - restart + - suspend + - resume + - remove + type: string + addPcieToAllowList: + description: restart params + items: + type: string + type: array + clusterImage: + type: string + clusterName: + type: string + coreIsolation: + type: boolean + coreMask: + type: string + corePercentage: + format: int32 + type: integer + dataNIC: + items: + type: string + type: array + driveSizeRange: + type: string + force: + type: boolean + haJM: + type: boolean + haJmCount: + format: int32 + type: integer + idDeviceByNQN: + type: boolean + jmPercent: + format: int32 + type: integer + maxLVol: + format: int32 + type: integer + maxSize: + type: string + mgmtIfc: + type: string + nodeAddr: + type: string + nodeUUID: + description: NodeUUID is required when action is specified + type: string + nodesPerSocket: + format: int32 + type: integer + openShiftCluster: + type: boolean + partitions: + format: int32 + type: integer + pcieAllowList: + items: + type: string + type: array + pcieDenyList: + items: + type: string + type: array + pcieModel: + type: string + socketsToUse: + format: int32 + type: integer + spdkDebug: + type: boolean + spdkImage: + type: string + useSeparateJournalDevice: + type: boolean + workerNode: + type: string + workerNodes: + items: + type: string + type: array + required: + - clusterName + type: object + status: + description: status defines the observed state of StorageNode + properties: + actionStatus: + properties: + action: + type: string + message: + type: string + nodeUUID: + type: string + observedGeneration: + format: int64 + type: integer + state: + type: string + updatedAt: + format: date-time + type: string + type: object + nodes: + items: + properties: + cpu: + format: int32 + type: integer + devices: + type: string + health: + type: boolean + hostname: + type: string + lvol_port: + format: int32 + type: integer + memory: + type: string + mgmtIp: + type: string + nvmf_port: + format: int32 + type: integer + rpc_port: + format: int32 + type: integer + status: + type: string + uptime: + type: string + uuid: + type: string + volumes: + format: int32 + type: integer + type: object + type: array + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: nodeUUID is required when action is specified + rule: '!(has(self.spec.action) && self.spec.action != "" && (!has(self.spec.nodeUUID) + || self.spec.nodeUUID == ""))' + - message: clusterImage, maxLVol, and workerNodes are required when action + is not specified + rule: (has(self.spec.action) && self.spec.action != "") || (has(self.spec.clusterImage) + && self.spec.clusterImage != "" && has(self.spec.maxLVol) && has(self.spec.workerNodes) + && size(self.spec.workerNodes) > 0) + served: true + storage: true + subresources: + status: {} diff --git a/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblocktasks.yaml b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblocktasks.yaml new file mode 100644 index 000000000..2d25e21e1 --- /dev/null +++ b/simplyblock_core/scripts/charts/crds/simplyblock.simplyblock.io_simplyblocktasks.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: simplyblocktasks.simplyblock.simplyblock.io +spec: + group: simplyblock.simplyblock.io + names: + kind: SimplyBlockTask + listKind: SimplyBlockTaskList + plural: simplyblocktasks + singular: simplyblocktask + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SimplyBlockTask is the Schema for the simplyblocktasks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SimplyBlockTask + properties: + clusterName: + type: string + subtasks: + type: boolean + taskID: + type: string + required: + - clusterName + type: object + status: + description: status defines the observed state of SimplyBlockTask + properties: + tasks: + items: + properties: + canceled: + type: boolean + parentTask: + type: string + retried: + format: int32 + type: integer + startedAt: + format: date-time + type: string + taskResult: + type: string + taskStatus: + type: string + taskType: + type: string + uuid: + type: string + type: object + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/simplyblock_core/scripts/charts/templates/_helpers.tpl b/simplyblock_core/scripts/charts/templates/_helpers.tpl new file mode 100644 index 000000000..710260fdc --- /dev/null +++ b/simplyblock_core/scripts/charts/templates/_helpers.tpl @@ -0,0 +1,21 @@ +{{- define "simplyblock.commonContainer" }} +env: + - name: SIMPLYBLOCK_LOG_LEVEL + valueFrom: + configMapKeyRef: + name: simplyblock-config + key: LOG_LEVEL + +volumeMounts: + - name: fdb-cluster-file + mountPath: /etc/foundationdb/fdb.cluster + subPath: fdb.cluster + +resources: + requests: + cpu: "50m" + memory: "100Mi" + limits: + cpu: "300m" + memory: "1Gi" +{{- end }} diff --git a/simplyblock_core/scripts/charts/templates/app_configmap.yaml b/simplyblock_core/scripts/charts/templates/app_configmap.yaml index de0a4da08..a4d1d57dd 100644 --- a/simplyblock_core/scripts/charts/templates/app_configmap.yaml +++ b/simplyblock_core/scripts/charts/templates/app_configmap.yaml @@ -6,8 +6,8 @@ metadata: namespace: {{ .Release.Namespace }} data: - LOG_LEVEL: {{ .Values.log.level }} - LOG_DELETION_INTERVAL: {{ .Values.log.deletionInterval }} + LOG_LEVEL: {{ .Values.observability.level }} + LOG_DELETION_INTERVAL: {{ .Values.observability.deletionInterval }} --- @@ -29,6 +29,7 @@ data: Path /var/log/containers/*.log Parser docker Tag kube.* + Exclude_Path /var/log/containers/*fluent-bit*.log Refresh_Interval 5 Mem_Buf_Limit 5MB Skip_Long_Lines On @@ -69,9 +70,11 @@ data: filter.lua: | function filter_tagged_pods(tag, timestamp, record) - annotations = record["kubernetes"]["annotations"] - if annotations ~= nil and annotations["log-collector/enabled"] == "true" then - return 1, record + if record["kubernetes"] ~= nil then + local annotations = record["kubernetes"]["annotations"] + if annotations ~= nil and annotations["log-collector/enabled"] == "true" then + return 1, record + end end return -1, record end diff --git a/simplyblock_core/scripts/charts/templates/app_ingress.yaml b/simplyblock_core/scripts/charts/templates/app_ingress.yaml index 67e7b0912..b49b0c396 100644 --- a/simplyblock_core/scripts/charts/templates/app_ingress.yaml +++ b/simplyblock_core/scripts/charts/templates/app_ingress.yaml @@ -1,4 +1,5 @@ -{{- if (not .Values.ingress.useDNS) }} +{{- if .Values.ingress.enabled }} + {{- if not .Values.ingress.useDNS }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -17,7 +18,7 @@ spec: port: number: 5000 --- -{{- else if .Values.ingress.useDNS }} + {{- else if .Values.ingress.useDNS }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -45,4 +46,5 @@ spec: name: simplyblock-webappapi port: number: 5000 + {{- end }} {{- end }} diff --git a/simplyblock_core/scripts/charts/templates/app_k8s.yaml b/simplyblock_core/scripts/charts/templates/app_k8s.yaml index 988955a4f..1626292e2 100644 --- a/simplyblock_core/scripts/charts/templates/app_k8s.yaml +++ b/simplyblock_core/scripts/charts/templates/app_k8s.yaml @@ -18,7 +18,7 @@ spec: labels: app: simplyblock-admin-control spec: - serviceAccountName: simplyblock-control-sa + serviceAccountName: simplyblock-sa hostNetwork: true dnsPolicy: ClusterFirstWithHostNet containers: @@ -33,11 +33,13 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace +{{- if .Values.observability.enabled }} - name: MONITORING_SECRET valueFrom: secretKeyRef: name: simplyblock-grafana-secrets key: MONITORING_SECRET +{{- end }} - name: SIMPLYBLOCK_LOG_LEVEL valueFrom: configMapKeyRef: @@ -63,11 +65,12 @@ spec: path: fdb.cluster --- apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: simplyblock-webappapi namespace: {{ .Release.Namespace }} spec: + replicas: 2 selector: matchLabels: app: simplyblock-webappapi @@ -79,7 +82,15 @@ spec: reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" labels: app: simplyblock-webappapi - spec: + spec: + serviceAccountName: simplyblock-sa + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: simplyblock-admin-control + topologyKey: kubernetes.io/hostname containers: - name: webappapi image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" @@ -95,15 +106,19 @@ spec: key: LOG_LEVEL - name: LVOL_NVMF_PORT_START value: "{{ .Values.ports.lvolNvmfPortStart }}" + - name: ENABLE_MONITORING + value: "{{ .Values.observability.enabled }}" - name: K8S_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace +{{- if .Values.observability.enabled }} - name: MONITORING_SECRET valueFrom: secretKeyRef: name: simplyblock-grafana-secrets key: MONITORING_SECRET +{{- end }} - name: FLASK_DEBUG value: "False" - name: FLASK_ENV @@ -119,57 +134,20 @@ spec: limits: cpu: "500m" memory: "2Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-storage-node-monitor - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-storage-node-monitor - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-storage-node-monitor - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: storage-node-monitor - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/storage_node_monitor.py"] - env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL + - name: fluent-bit + image: fluent/fluent-bit:1.8.11 volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster + - name: varlog + mountPath: /var/log + - name: config + mountPath: /fluent-bit/etc/ resources: requests: - cpu: "200m" - memory: "256Mi" + cpu: "100m" + memory: "200Mi" limits: - cpu: "400m" - memory: "1Gi" + cpu: "200m" + memory: "400Mi" volumes: - name: fdb-cluster-file configMap: @@ -177,18 +155,23 @@ spec: items: - key: cluster-file path: fdb.cluster - + - name: varlog + hostPath: + path: /var/log + - name: config + configMap: + name: simplyblock-fluent-bit-config --- apiVersion: apps/v1 kind: Deployment metadata: - name: simplyblock-mgmt-node-monitor + name: simplyblock-monitoring namespace: {{ .Release.Namespace }} spec: replicas: 1 selector: matchLabels: - app: simplyblock-mgmt-node-monitor + app: simplyblock-monitoring template: metadata: annotations: @@ -196,261 +179,172 @@ spec: reloader.stakater.com/auto: "true" reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" labels: - app: simplyblock-mgmt-node-monitor + app: simplyblock-monitoring spec: + serviceAccountName: simplyblock-sa hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirstWithHostNet containers: + - name: storage-node-monitor + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + command: ["python", "simplyblock_core/services/storage_node_monitor.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} + env: +{{ toYaml .env | nindent 12 }} + volumeMounts: +{{ toYaml .volumeMounts | nindent 12 }} + resources: +{{ toYaml .resources | nindent 12 }} +{{- end }} + - name: mgmt-node-monitor image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/mgmt_node_monitor.py"] env: - - name: BACKEND_TYPE - value: "k8s" - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL + - name: BACKEND_TYPE + value: "k8s" +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster +{{ toYaml .resources | nindent 12 }} +{{- end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-lvol-stats-collector - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-lvol-stats-collector - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-lvol-stats-collector - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - name: lvol-stats-collector image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/lvol_stat_collector.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster +{{ toYaml .resources | nindent 12 }} +{{- end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-main-distr-event-collector - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-main-distr-event-collector - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-main-distr-event-collector - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - name: main-distr-event-collector image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/main_distr_event_collector.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} + env: +{{ toYaml .env | nindent 12 }} + volumeMounts: +{{ toYaml .volumeMounts | nindent 12 }} + resources: +{{ toYaml .resources | nindent 12 }} +{{- end }} + + - name: capacity-and-stats-collector + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + command: ["python", "simplyblock_core/services/capacity_and_stats_collector.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} + env: +{{ toYaml .env | nindent 12 }} + volumeMounts: +{{ toYaml .volumeMounts | nindent 12 }} + resources: +{{ toYaml .resources | nindent 12 }} +{{- end }} + + - name: capacity-monitor + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + command: ["python", "simplyblock_core/services/cap_monitor.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} + env: +{{ toYaml .env | nindent 12 }} + volumeMounts: +{{ toYaml .volumeMounts | nindent 12 }} + resources: +{{ toYaml .resources | nindent 12 }} +{{- end }} + + - name: health-check + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + command: ["python", "simplyblock_core/services/health_check_service.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} + env: +{{ toYaml .env | nindent 12 }} + volumeMounts: +{{ toYaml .volumeMounts | nindent 12 }} + resources: +{{ toYaml .resources | nindent 12 }} +{{- end }} + + - name: device-monitor + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + command: ["python", "simplyblock_core/services/device_monitor.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} + env: +{{ toYaml .env | nindent 12 }} + volumeMounts: +{{ toYaml .volumeMounts | nindent 12 }} + resources: +{{ toYaml .resources | nindent 12 }} +{{- end }} + + - name: lvol-monitor + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + command: ["python", "simplyblock_core/services/lvol_monitor.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} + volumeMounts: +{{ toYaml .volumeMounts | nindent 12 }} + resources: +{{ toYaml .resources | nindent 12 }} +{{- end }} + + - name: snapshot-monitor + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + command: ["python", "simplyblock_core/services/snapshot_monitor.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} + env: +{{ toYaml .env | nindent 12 }} + volumeMounts: +{{ toYaml .volumeMounts | nindent 12 }} + resources: +{{ toYaml .resources | nindent 12 }} +{{- end }} + - name: fluent-bit + image: fluent/fluent-bit:1.8.11 volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster + - name: varlog + mountPath: /var/log + - name: config + mountPath: /fluent-bit/etc/ resources: requests: - cpu: "200m" - memory: "256Mi" + cpu: "100m" + memory: "200Mi" limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster + cpu: "200m" + memory: "400Mi" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-capacity-and-stats-collector - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-capacity-and-stats-collector - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-capacity-and-stats-collector - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: capacity-and-stats-collector - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/capacity_and_stats_collector.py"] - env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster - resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-capacity-monitor - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-capacity-monitor - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-capacity-monitor - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: capacity-monitor - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/cap_monitor.py"] - env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - volumeMounts: - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster - resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster - + configMap: + name: simplyblock-fdb-cluster-config + items: + - key: cluster-file + path: fdb.cluster + - name: varlog + hostPath: + path: /var/log + - name: config + configMap: + name: simplyblock-fluent-bit-config --- apiVersion: apps/v1 kind: Deployment metadata: - name: simplyblock-health-check + name: simplyblock-tasks namespace: {{ .Release.Namespace }} spec: replicas: 1 selector: matchLabels: - app: simplyblock-health-check + app: simplyblock-tasks template: metadata: annotations: @@ -458,754 +352,146 @@ spec: reloader.stakater.com/auto: "true" reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" labels: - app: simplyblock-health-check + app: simplyblock-tasks spec: + serviceAccountName: simplyblock-sa hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirstWithHostNet containers: - - name: health-check - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/health_check_service.py"] - env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster - resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-device-monitor - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-device-monitor - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-device-monitor - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: device-monitor - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/device_monitor.py"] - env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster - resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-lvol-monitor - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-lvol-monitor - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-lvol-monitor - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: lvol-monitor - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/lvol_monitor.py"] - env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster - resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-snapshot-monitor - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-snapshot-monitor - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-snapshot-monitor - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: snapshot-monitor - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/snapshot_monitor.py"] - env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster - resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-cleanupfdb - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-cleanupfdb - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-cleanupfdb - spec: - containers: - - name: cleanupfdb + - name: tasks-node-add-runner image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/workers/cleanup_foundationdb.py"] + command: ["python", "simplyblock_core/services/tasks_runner_node_add.py"] env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - - name: LOG_DELETION_INTERVAL - value: "${LOG_DELETION_INTERVAL}" + - name: LVOL_NVMF_PORT_START + value: "{{ .Values.ports.lvolNvmfPortStart }}" +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster +{{ toYaml .resources | nindent 12 }} +{{- end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-runner-restart - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-runner-restart - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-runner-restart - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - name: tasks-runner-restart image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/tasks_runner_restart.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster +{{ toYaml .resources | nindent 12 }} +{{- end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-runner-migration - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-runner-migration - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-runner-migration - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - name: tasks-runner-migration image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/tasks_runner_migration.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-runner-failed-migration - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-runner-failed-migration - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-runner-failed-migration - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: +{{ toYaml .resources | nindent 12 }} +{{- end }} + - name: tasks-runner-failed-migration image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/tasks_runner_failed_migration.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-runner-cluster-status - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-runner-cluster-status - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-runner-cluster-status - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: +{{ toYaml .resources | nindent 12 }} +{{- end }} + - name: tasks-runner-cluster-status image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/tasks_cluster_status.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-runner-new-device-migration - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-runner-new-device-migration - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-runner-new-device-migration - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: +{{ toYaml .resources | nindent 12 }} +{{- end }} + - name: tasks-runner-new-device-migration image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/tasks_runner_new_dev_migration.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL - volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster - resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-node-add-runner - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-node-add-runner - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-node-add-runner - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: tasks-node-addrunner - image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" - command: ["python", "simplyblock_core/services/tasks_runner_node_add.py"] - env: - - name: LVOL_NVMF_PORT_START - value: "{{ .Values.ports.lvolNvmfPortStart }}" - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster +{{ toYaml .resources | nindent 12 }} +{{- end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-runner-port-allow - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-runner-port-allow - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-runner-port-allow - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - name: tasks-runner-port-allow image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/tasks_runner_port_allow.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-runner-jc-comp-resume - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-runner-jc-comp-resume - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-runner-jc-comp-resume - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: +{{ toYaml .resources | nindent 12 }} +{{- end }} + - name: tasks-runner-jc-comp-resume image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/tasks_runner_jc_comp.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-tasks-runner-sync-lvol-del - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-tasks-runner-sync-lvol-del - template: - metadata: - annotations: - log-collector/enabled: "true" - reloader.stakater.com/auto: "true" - reloader.stakater.com/configmap: "simplyblock-fdb-cluster-config" - labels: - app: simplyblock-tasks-runner-sync-lvol-del - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: +{{ toYaml .resources | nindent 12 }} +{{- end }} + - name: tasks-runner-sync-lvol-del image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" - imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" command: ["python", "simplyblock_core/services/tasks_runner_sync_lvol_del.py"] +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} env: - - name: SIMPLYBLOCK_LOG_LEVEL - valueFrom: - configMapKeyRef: - name: simplyblock-config - key: LOG_LEVEL +{{ toYaml .env | nindent 12 }} volumeMounts: - - name: fdb-cluster-file - mountPath: /etc/foundationdb/fdb.cluster - subPath: fdb.cluster +{{ toYaml .volumeMounts | nindent 12 }} resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "400m" - memory: "1Gi" - volumes: - - name: fdb-cluster-file - configMap: - name: simplyblock-fdb-cluster-config - items: - - key: cluster-file - path: fdb.cluster ---- - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: simplyblock-fluent-bit - namespace: {{ .Release.Namespace }} - labels: - app: simplyblock-fluent-bit -spec: - selector: - matchLabels: - app: simplyblock-fluent-bit - template: - metadata: - labels: - app: simplyblock-fluent-bit - spec: - containers: +{{ toYaml .resources | nindent 12 }} +{{- end }} - name: fluent-bit image: fluent/fluent-bit:1.8.11 - securityContext: - privileged: true volumeMounts: - name: varlog mountPath: /var/log - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers - readOnly: true - name: config mountPath: /fluent-bit/etc/ resources: requests: + cpu: "100m" + memory: "200Mi" + limits: cpu: "200m" memory: "400Mi" - limits: - cpu: "400m" - memory: "1Gi" + volumes: + - name: fdb-cluster-file + configMap: + name: simplyblock-fdb-cluster-config + items: + - key: cluster-file + path: fdb.cluster - name: varlog hostPath: path: /var/log - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers - name: config configMap: name: simplyblock-fluent-bit-config diff --git a/simplyblock_core/scripts/charts/templates/app_sa.yaml b/simplyblock_core/scripts/charts/templates/app_sa.yaml index a5dee735b..f04fc14b3 100644 --- a/simplyblock_core/scripts/charts/templates/app_sa.yaml +++ b/simplyblock_core/scripts/charts/templates/app_sa.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: simplyblock-control-sa + name: simplyblock-sa namespace: {{ .Release.Namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: simplyblock-control-role + name: simplyblock-role rules: - apiGroups: [""] resources: ["configmaps"] @@ -21,16 +21,23 @@ rules: - apiGroups: ["mongodbcommunity.mongodb.com"] resources: ["mongodbcommunity"] verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: ["simplyblock.simplyblock.io"] + resources: ["simplyblockpools/status", "simplyblocklvols/status", "simplyblockstorageclusters/status", "simplyblockstoragenodes/status", "simplyblockdevices/status", "simplyblocktasks/status"] + verbs: ["get", "patch", "update"] + - apiGroups: ["simplyblock.simplyblock.io"] + resources: ["simplyblockpools", "simplyblocklvols", "simplyblockstorageclusters", "simplyblockstoragenodes", "simplyblockdevices", "simplyblocktasks"] + verbs: ["get","list" ,"patch", "update", "watch"] + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: simplyblock-control-binding + name: simplyblock-binding subjects: - kind: ServiceAccount - name: simplyblock-control-sa + name: simplyblock-sa namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole - name: simplyblock-control-role + name: simplyblock-role apiGroup: rbac.authorization.k8s.io diff --git a/simplyblock_core/scripts/charts/templates/csi-hostpath-plugin.yaml b/simplyblock_core/scripts/charts/templates/csi-hostpath-plugin.yaml index 8e695e593..721815fa5 100644 --- a/simplyblock_core/scripts/charts/templates/csi-hostpath-plugin.yaml +++ b/simplyblock_core/scripts/charts/templates/csi-hostpath-plugin.yaml @@ -158,8 +158,6 @@ spec: - --v=5 - --endpoint=$(CSI_ENDPOINT) - --nodeid=$(KUBE_NODE_NAME) - - --capacity=slow=10Gi - - --capacity=fast=100Gi env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock diff --git a/simplyblock_core/scripts/charts/templates/dashboards.yaml b/simplyblock_core/scripts/charts/templates/dashboards.yaml index 981e961d0..165bad130 100644 --- a/simplyblock_core/scripts/charts/templates/dashboards.yaml +++ b/simplyblock_core/scripts/charts/templates/dashboards.yaml @@ -1,4 +1,4 @@ -{{- if .Values.monitoring.enabled }} +{{- if .Values.observability.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -12512,14796 +12512,4 @@ data: "weekStart": "" } ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: simplyblock-grafana-dashboard-node-exporter - namespace: {{ .Release.Namespace }} - labels: - grafana_dashboard: "1" -data: - node-exporter.json: | - { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 1860, - "graphTooltip": 1, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 261, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "Quick CPU / Mem / Disk", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Resource pressure via PSI", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "links": [], - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "dark-yellow", - "value": 70 - }, - { - "color": "dark-red", - "value": 90 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 323, - "links": [], - "options": { - "displayMode": "basic", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": {} - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "irate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "legendFormat": "CPU", - "range": false, - "refId": "CPU some", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "irate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "Mem", - "range": false, - "refId": "Memory some", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "irate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "I/O", - "range": false, - "refId": "I/O some", - "step": 240 - } - ], - "title": "Pressure", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Busy state of all CPU cores together", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 85 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 20, - "links": [], - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", instance=\"$node\"}[$__rate_interval])))", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "range": false, - "refId": "A", - "step": 240 - } - ], - "title": "CPU Busy", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "System load over all CPU cores together", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 85 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 1 - }, - "id": 155, - "links": [], - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "scalar(node_load1{instance=\"$node\",job=\"$job\"}) * 100 / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "A", - "step": 240 - } - ], - "title": "Sys Load", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Non available RAM memory", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 1 - }, - "hideTimeOverride": false, - "id": 16, - "links": [], - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\", job=\"$job\"}) / node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"}) * 100", - "format": "time_series", - "hide": true, - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(1 - (node_memory_MemAvailable_bytes{instance=\"$node\", job=\"$job\"} / node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"})) * 100", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "B", - "step": 240 - } - ], - "title": "RAM Used", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Used Swap", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 10 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 25 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 21, - "links": [], - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"})) * 100", - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "A", - "step": 240 - } - ], - "title": "SWAP Used", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Used Root FS", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 1 - }, - "id": 154, - "links": [], - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "A", - "step": 240 - } - ], - "title": "Root FS Used", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of CPU cores", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 2, - "x": 18, - "y": 1 - }, - "id": 14, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "CPU Cores", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "System uptime", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 20, - "y": 1 - }, - "hideTimeOverride": true, - "id": 15, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "A", - "step": 240 - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total RootFS", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 70 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 2, - "x": 18, - "y": 3 - }, - "id": 23, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "A", - "step": 240 - } - ], - "title": "RootFS Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total RAM", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 2, - "x": 20, - "y": 3 - }, - "id": 75, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "A", - "step": 240 - } - ], - "title": "RAM Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total SWAP", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 2, - "x": 22, - "y": 3 - }, - "id": 18, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "A", - "step": 240 - } - ], - "title": "SWAP Total", - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 263, - "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "Basic CPU / Mem / Net / Disk", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Basic CPU info", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Busy Iowait" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Idle" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Busy Iowait" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Idle" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Busy System" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Busy User" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Busy Other" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 77, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "width": 250 - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "hide": false, - "instant": false, - "intervalFactor": 1, - "legendFormat": "Busy System", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Busy User", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Busy Iowait", - "range": true, - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Busy IRQs", - "range": true, - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Busy Other", - "range": true, - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Idle", - "range": true, - "refId": "F", - "step": 240 - } - ], - "title": "CPU Basic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Basic memory usage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "SWAP Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Cache + Buffer" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Available" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#DEDAF7", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 78, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "RAM Total", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "RAM Used", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "RAM Cache + Buffer", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "RAM Free", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SWAP Used", - "refId": "E", - "step": 240 - } - ], - "title": "Memory Basic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Basic network info per interface", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Recv_bytes_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_drop_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_errs_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_bytes_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_drop_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_errs_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_drop_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#967302", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_errs_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_bytes_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_drop_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#967302", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_errs_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 74, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "recv {{ "{{" }}device{{ "}}" }}", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "trans {{ "{{" }}device{{ "}}" }} ", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Basic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Disk space used of all filesystems mounted", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 152, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}mountpoint{{ "}}" }}", - "refId": "A", - "step": 240 - } - ], - "title": "Disk Space Used Basic", - "type": "timeseries" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 265, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "percentage", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 70, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Idle - Waiting for something to happen" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Iowait - Waiting for I/O to complete" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Irq - Servicing interrupts" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Nice - Niced processes executing in user mode" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Softirq - Servicing softirqs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Steal - Time spent in other operating systems when running in a virtualized environment" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCE2DE", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "System - Processes executing in kernel mode" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "User - Normal processes executing in user mode" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#5195CE", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 3, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 250 - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "System - Processes executing in kernel mode", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "User - Normal processes executing in user mode", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Nice - Niced processes executing in user mode", - "range": true, - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Iowait - Waiting for I/O to complete", - "range": true, - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Irq - Servicing interrupts", - "range": true, - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Softirq - Servicing softirqs", - "range": true, - "refId": "G", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", - "range": true, - "refId": "H", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Idle - Waiting for something to happen", - "range": true, - "refId": "J", - "step": 240 - } - ], - "title": "CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap - Swap memory usage" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused - Free memory unassigned" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Hardware Corrupted - *./" - }, - "properties": [ - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 24, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Apps - Memory used by user-space applications", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Cache - Parked file data (file content) cache", - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Buffers - Block device (e.g. harddisk) cache", - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Unused - Free memory unassigned", - "refId": "G", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Swap - Swap space used", - "refId": "H", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", - "refId": "I", - "step": 240 - } - ], - "title": "Memory Stack", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bits out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "receive_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "receive_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 84, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Receive", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Transmit", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 156, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}mountpoint{{ "}}" }}", - "refId": "A", - "step": 240 - } - ], - "title": "Disk Space Used", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "IO read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 45 - }, - "id": 229, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "intervalFactor": 4, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Reads completed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Writes completed", - "refId": "B", - "step": 240 - } - ], - "title": "Disk IOps", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "io time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*read*./" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byType", - "options": "time" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 45 - }, - "id": 42, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Successfully read bytes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Successfully written bytes", - "refId": "B", - "step": 240 - } - ], - "title": "I/O Usage Read / Write", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "%util", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "io time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byType", - "options": "time" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 57 - }, - "id": 127, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }}", - "refId": "A", - "step": 240 - } - ], - "title": "I/O Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "percentage", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 70, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^Guest - /" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#5195ce", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^GuestNice - /" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#c15c17", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 57 - }, - "id": 319, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", - "hide": false, - "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", - "hide": false, - "legendFormat": "GuestNice - Time spent running a niced guest (virtual CPU for guest operating system)", - "range": true, - "refId": "B" - } - ], - "title": "CPU spent seconds in guests (VMs)", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "CPU / Memory / Net / Disk", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 266, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 54 - }, - "id": 136, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Active / Inactive", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*CommitLimit - *./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 54 - }, - "id": 135, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Committed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 191, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Active_file - File-backed memory on active LRU list", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", - "refId": "D", - "step": 240 - } - ], - "title": "Memory Active / Inactive Detail", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 64 - }, - "id": 130, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Writeback - Memory which is actively being written back to disk", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", - "refId": "C", - "step": 240 - } - ], - "title": "Memory Writeback and Dirty", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 74 - }, - "id": 138, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages", - "refId": "D", - "step": 240 - } - ], - "title": "Memory Shared and Mapped", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 74 - }, - "id": 131, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Slab", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 84 - }, - "id": 70, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocChunk - Largest contiguous block of vmalloc area which is free", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocTotal - Total size of vmalloc memory area", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", - "refId": "C", - "step": 240 - } - ], - "title": "Memory Vmalloc", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 84 - }, - "id": 159, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Bounce - Memory used for block device bounce buffers", - "refId": "A", - "step": 240 - } - ], - "title": "Memory Bounce", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Inactive *./" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 94 - }, - "id": 129, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "AnonHugePages - Memory in anonymous huge pages", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "AnonPages - Memory in user pages not backed by files", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Anonymous", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 94 - }, - "id": 160, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Kernel / CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "pages", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 104 - }, - "id": 140, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", - "refId": "C", - "step": 240 - } - ], - "title": "Memory HugePages Counter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 104 - }, - "id": 71, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages - Total size of the pool of huge pages", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Hugepagesize - Huge Page size", - "refId": "B", - "step": 240 - } - ], - "title": "Memory HugePages Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 114 - }, - "id": 128, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "DirectMap1G - Amount of pages mapped as this size", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "DirectMap2M - Amount of pages mapped as this size", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "DirectMap4K - Amount of pages mapped as this size", - "refId": "C", - "step": 240 - } - ], - "title": "Memory DirectMap", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 114 - }, - "id": 137, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Unevictable and MLocked", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 124 - }, - "id": 132, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage", - "refId": "A", - "step": 240 - } - ], - "title": "Memory NFS", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "Memory Meminfo", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 270, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "The number (after merges) of I/O requests completed per second for the device", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "IO read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 9, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 4, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Reads completed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Writes completed", - "refId": "B", - "step": 240 - } - ], - "title": "Disk IOps Completed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "The number of bytes read from or written to the device per second", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 33, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 4, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Read bytes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Written bytes", - "refId": "B", - "step": 240 - } - ], - "title": "Disk R/W Data", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "time. read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 57 - }, - "id": 37, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Read wait time avg", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Write wait time avg", - "refId": "B", - "step": 240 - } - ], - "title": "Disk Average Wait Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "The average queue length of the requests that were issued to the device", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "aqu-sz", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 57 - }, - "id": 35, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{ "{{" }}device{{ "}}" }}", - "refId": "A", - "step": 240 - } - ], - "title": "Average Queue Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "The number of read and write requests merged per second that were queued to the device", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "I/Os", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 133, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Read merged", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Write merged", - "refId": "B", - "step": 240 - } - ], - "title": "Disk R/W Merged", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "%util", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 36, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - IO", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - discard", - "refId": "B", - "step": 240 - } - ], - "title": "Time Spent Doing I/Os", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Outstanding req.", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 77 - }, - "id": 34, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - IO now", - "refId": "A", - "step": 240 - } - ], - "title": "Instantaneous Queue Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "IOs", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 77 - }, - "id": 301, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Discards completed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}device{{ "}}" }} - Discards merged", - "refId": "B", - "step": 240 - } - ], - "title": "Disk IOps Discards completed / merged", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "Storage Disk", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 271, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 62 - }, - "id": 43, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}mountpoint{{ "}}" }} - Available", - "metric": "", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}mountpoint{{ "}}" }} - Free", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}mountpoint{{ "}}" }} - Size", - "refId": "C", - "step": 240 - } - ], - "title": "Filesystem space available", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "file nodes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 62 - }, - "id": 41, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}mountpoint{{ "}}" }} - Free file nodes", - "refId": "A", - "step": 240 - } - ], - "title": "File Nodes Free", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "files", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 72 - }, - "id": 28, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 4, - "legendFormat": "Max open files", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Open files", - "refId": "B", - "step": 240 - } - ], - "title": "File Descriptor", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "file Nodes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 72 - }, - "id": 219, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}mountpoint{{ "}}" }} - File nodes total", - "refId": "A", - "step": 240 - } - ], - "title": "File Nodes Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "/ ReadOnly" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 82 - }, - "id": 44, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}mountpoint{{ "}}" }} - ReadOnly", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}mountpoint{{ "}}" }} - Device error", - "refId": "B", - "step": 240 - } - ], - "title": "Filesystem in ReadOnly / Error", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "Storage Filesystem", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 279, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 66 - }, - "id": 40, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}collector{{ "}}" }} - Scrape duration", - "refId": "A", - "step": 240 - } - ], - "title": "Node Exporter Scrape Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*error.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - }, - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 66 - }, - "id": 157, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}collector{{ "}}" }} - Scrape success", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ "{{" }}collector{{ "}}" }} - Scrape textfile error (1 = true)", - "refId": "B", - "step": 240 - } - ], - "title": "Node Exporter Scrape", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "Node Exporter", - "type": "row" - } - ], - "refresh": "1m", - "revision": 1, - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Datasource", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "definition": "", - "hide": 0, - "includeAll": false, - "label": "Job", - "multi": false, - "name": "job", - "options": [], - "query": { - "query": "label_values(node_uname_info, job)", - "refId": "Prometheus-job-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "definition": "label_values(node_uname_info{job=\"$job\"}, instance)", - "hide": 0, - "includeAll": false, - "label": "Host", - "multi": false, - "name": "node", - "options": [], - "query": { - "query": "label_values(node_uname_info{job=\"$job\"}, instance)", - "refId": "Prometheus-node-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" - }, - "hide": 2, - "includeAll": false, - "multi": false, - "name": "diskdevices", - "options": [ - { - "selected": true, - "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" - } - ], - "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "NodeExporter", - "uid": "d56e0ae7-48d5-481d-a2ea-3192da4d9e42", - "version": 5, - "weekStart": "" - } {{- end }} diff --git a/simplyblock_core/scripts/charts/templates/foundationdb.yaml b/simplyblock_core/scripts/charts/templates/foundationdb.yaml index 4eb7f1410..5020c2fea 100644 --- a/simplyblock_core/scripts/charts/templates/foundationdb.yaml +++ b/simplyblock_core/scripts/charts/templates/foundationdb.yaml @@ -246,7 +246,6 @@ spec: general: customParameters: - knob_disable_posix_kernel_aio=1 - - listen_address=0.0.0.0:4501 podTemplate: spec: containers: diff --git a/simplyblock_core/scripts/charts/templates/mongodb.yaml b/simplyblock_core/scripts/charts/templates/mongodb.yaml index 815df6505..6c004f314 100644 --- a/simplyblock_core/scripts/charts/templates/mongodb.yaml +++ b/simplyblock_core/scripts/charts/templates/mongodb.yaml @@ -1,3 +1,4 @@ +{{- if .Values.observability.enabled }} apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: @@ -51,4 +52,5 @@ metadata: name: admin-password type: Opaque stringData: - password: {{ .Values.monitoring.secret }} + password: {{ .Values.observability.secret }} +{{- end }} diff --git a/simplyblock_core/scripts/charts/templates/monitoring_configmap.yaml b/simplyblock_core/scripts/charts/templates/monitoring_configmap.yaml index cb4243493..497760180 100644 --- a/simplyblock_core/scripts/charts/templates/monitoring_configmap.yaml +++ b/simplyblock_core/scripts/charts/templates/monitoring_configmap.yaml @@ -1,4 +1,3 @@ -{{- if .Values.monitoring.enabled }} apiVersion: v1 kind: ConfigMap @@ -25,14 +24,6 @@ data: username: password: - - job_name: 'node' - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - source_labels: [__meta_kubernetes_endpoints_name] - action: keep - regex: 'simplyblock-node-exporter' - --- apiVersion: v1 kind: ConfigMap @@ -46,6 +37,7 @@ data: type: FILESYSTEM config: directory: /mnt/thanos +{{- if .Values.observability.enabled }} --- apiVersion: v1 kind: ConfigMap @@ -60,7 +52,7 @@ data: datasources: - name: Thanos type: prometheus - url: http://simplyblock-thanos-query:9091 + url: http://simplyblock-thanos:9091 isDefault: true access: proxy uid: PBFA97CFB590B2093 @@ -829,7 +821,7 @@ data: type: slack settings: username: grafana_bot - url: '{{ .Values.grafana.contactPoint }}' + url: '{{ .Values.observability.grafana.contactPoint }}' title: | '{{ "{{" }} template "slack.title" . {{ "}}" }}' text: | diff --git a/simplyblock_core/scripts/charts/templates/monitoring_ingress.yaml b/simplyblock_core/scripts/charts/templates/monitoring_ingress.yaml index ec0e1ab80..bcccf4a35 100644 --- a/simplyblock_core/scripts/charts/templates/monitoring_ingress.yaml +++ b/simplyblock_core/scripts/charts/templates/monitoring_ingress.yaml @@ -1,4 +1,5 @@ -{{- if (not .Values.ingress.useDNS) }} +{{- if .Values.ingress.enabled }} + {{- if not .Values.ingress.useDNS }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -32,9 +33,8 @@ spec: name: simplyblock-graylog port: number: 9000 - --- -{{- else if .Values.ingress.useDNS }} + {{- else if .Values.ingress.useDNS }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -77,4 +77,5 @@ spec: name: simplyblock-graylog port: number: 9000 + {{- end }} {{- end }} diff --git a/simplyblock_core/scripts/charts/templates/monitoring_k8s.yaml b/simplyblock_core/scripts/charts/templates/monitoring_k8s.yaml index 1349a33a9..f54a9c2f5 100644 --- a/simplyblock_core/scripts/charts/templates/monitoring_k8s.yaml +++ b/simplyblock_core/scripts/charts/templates/monitoring_k8s.yaml @@ -1,4 +1,4 @@ -{{- if .Values.monitoring.enabled }} +{{- if .Values.observability.enabled }} --- apiVersion: apps/v1 kind: Deployment @@ -46,7 +46,7 @@ spec: - name: GRAYLOG_ELASTICSEARCH_HOSTS value: "http://opensearch-cluster-master:9200" - name: GRAYLOG_MONGODB_URI - value: "mongodb://admin:{{ .Values.monitoring.secret }}@simplyblock-mongo-svc:27017/graylog" + value: "mongodb://admin:{{ .Values.observability.secret }}@simplyblock-mongo-svc:27017/graylog" - name: GRAYLOG_SKIP_PREFLIGHT_CHECKS value: "true" - name: GRAYLOG_ROTATION_STRATEGY @@ -105,30 +105,37 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: simplyblock-thanos-store + name: simplyblock-thanos namespace: {{ .Release.Namespace }} spec: replicas: 1 selector: matchLabels: - app: simplyblock-thanos-store + app: simplyblock-thanos template: metadata: labels: - app: simplyblock-thanos-store - spec: + app: simplyblock-thanos + spec: containers: - name: thanos-store image: thanosio/thanos:v0.31.0 args: - store + - --grpc-address=0.0.0.0:10901 + - --http-address=0.0.0.0:10902 - --objstore.config-file=/etc/thanos/objstore.yml - --index-cache-size=500MB - --chunk-pool-size=500MB + ports: + - name: grpc + containerPort: 10901 + - name: http + containerPort: 10902 volumeMounts: - name: objstore-config mountPath: /etc/thanos - - name: thanos-data + - name: data mountPath: /data resources: requests: @@ -137,37 +144,20 @@ spec: limits: cpu: "250m" memory: "1Gi" - volumes: - - name: objstore-config - configMap: - name: simplyblock-objstore-config - - name: thanos-data - emptyDir: {} - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-thanos-query - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-thanos-query - template: - metadata: - labels: - app: simplyblock-thanos-query - spec: - containers: + - name: thanos-query image: thanosio/thanos:v0.31.0 args: - query + - --grpc-address=0.0.0.0:10911 - --http-address=0.0.0.0:9091 - - --store=simplyblock-thanos-store:10901 + - --store=simplyblock-thanos:10901 - --store=simplyblock-prometheus:10901 + ports: + - containerPort: 9091 + name: http + - containerPort: 10911 + name: grpc resources: requests: cpu: "100m" @@ -176,28 +166,11 @@ spec: cpu: "250m" memory: "1Gi" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplyblock-thanos-compactor - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: simplyblock-thanos-compactor - template: - metadata: - labels: - app: simplyblock-thanos-compactor - spec: - - containers: - name: thanos-compactor image: thanosio/thanos:v0.31.0 args: - compact + - --http-address=0.0.0.0:10922 - --data-dir=/data - --objstore.config-file=/etc/thanos/objstore.yml - --retention.resolution-raw=30d @@ -205,10 +178,13 @@ spec: - --retention.resolution-1h=90d - --compact.concurrency=1 - --wait + ports: + - containerPort: 10922 + name: http volumeMounts: - name: objstore-config mountPath: /etc/thanos - - name: compactor-data + - name: data mountPath: /data resources: requests: @@ -217,72 +193,14 @@ spec: limits: cpu: "250m" memory: "1Gi" + volumes: - name: objstore-config configMap: name: simplyblock-objstore-config - - name: compactor-data - emptyDir: {} ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: simplyblock-node-exporter - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: simplyblock-node-exporter - template: - metadata: - labels: - app: simplyblock-node-exporter - spec: - containers: - - name: node-exporter - image: prom/node-exporter:v1.7.0 - args: - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - - '--path.rootfs=/host/root' - - '--collector.filesystem.ignored-mount-points=^(/rootfs|/host|)/(sys|proc|dev|host|etc|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/)' - - '--collector.filesystem.ignored-fs-types=^(sys|proc|auto|cgroup|devpts|ns|au|fuse.lxc|mqueue)(fs|)$' - - '--no-collector.ipvs' - - '--web.listen-address=:9200' - ports: - - containerPort: 9200 - protocol: TCP - volumeMounts: - - name: proc - mountPath: /host/proc - readOnly: true - mountPropagation: HostToContainer - - name: sys - mountPath: /host/sys - mountPropagation: HostToContainer - readOnly: true - - name: root - mountPath: /host/root - mountPropagation: HostToContainer - readOnly: true - resources: - requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "250m" - memory: "1Gi" - volumes: - - name: proc - hostPath: - path: /proc - - name: sys - hostPath: - path: /sys - - name: root - hostPath: - path: / + - name: data + emptyDir: {} --- apiVersion: apps/v1 @@ -345,9 +263,6 @@ spec: - name: dashboard-pools mountPath: /var/lib/grafana/dashboards/pools.json subPath: pools.json - - name: dashboard-node-exporter - mountPath: /var/lib/grafana/dashboards/node-exporter.json - subPath: node-exporter.json - name: grafana-data mountPath: /var/lib/grafana volumes: @@ -375,9 +290,6 @@ spec: - name: dashboard-pools configMap: name: simplyblock-grafana-dashboard-pools - - name: dashboard-node-exporter - configMap: - name: simplyblock-grafana-dashboard-node-exporter - name: grafana-data emptyDir: {} {{- end }} diff --git a/simplyblock_core/scripts/charts/templates/monitoring_secret.yaml b/simplyblock_core/scripts/charts/templates/monitoring_secret.yaml index c39735159..df741f026 100644 --- a/simplyblock_core/scripts/charts/templates/monitoring_secret.yaml +++ b/simplyblock_core/scripts/charts/templates/monitoring_secret.yaml @@ -1,4 +1,4 @@ -{{- if .Values.monitoring.enabled }} +{{- if .Values.observability.enabled }} apiVersion: v1 kind: Secret metadata: @@ -6,8 +6,8 @@ metadata: namespace: {{ .Release.Namespace }} type: Opaque stringData: - MONITORING_SECRET: "{{ .Values.monitoring.secret }}" - GRAFANA_ENDPOINT: "{{ .Values.grafana.endpoint }}" + MONITORING_SECRET: "{{ .Values.observability.secret }}" + GRAFANA_ENDPOINT: "{{ .Values.observability.grafana.endpoint }}" --- apiVersion: v1 @@ -17,7 +17,7 @@ metadata: namespace: {{ .Release.Namespace }} type: Opaque stringData: - GRAYLOG_PASSWORD_SECRET: "{{ .Values.graylog.passwordSecret }}" - GRAYLOG_ROOT_PASSWORD_SHA2: "{{ .Values.graylog.rootPasswordSha2 }}" - MAX_NUMBER_OF_INDICES: "{{ .Values.log.maxNumberIndex }}" + GRAYLOG_PASSWORD_SECRET: "{{ .Values.observability.graylog.passwordSecret }}" + GRAYLOG_ROOT_PASSWORD_SHA2: "{{ .Values.observability.graylog.rootPasswordSha2 }}" + MAX_NUMBER_OF_INDICES: "{{ .Values.observability.graylog.maxNumberIndex }}" {{- end }} diff --git a/simplyblock_core/scripts/charts/templates/monitoring_svc.yaml b/simplyblock_core/scripts/charts/templates/monitoring_svc.yaml index 55b15dccc..5a0936434 100644 --- a/simplyblock_core/scripts/charts/templates/monitoring_svc.yaml +++ b/simplyblock_core/scripts/charts/templates/monitoring_svc.yaml @@ -1,4 +1,4 @@ -{{- if .Values.monitoring.enabled }} +{{- if .Values.observability.enabled }} --- apiVersion: v1 kind: Service @@ -25,44 +25,19 @@ spec: apiVersion: v1 kind: Service metadata: - name: simplyblock-thanos-store + name: simplyblock-thanos namespace: {{ .Release.Namespace }} spec: selector: - app: simplyblock-thanos-store + app: simplyblock-thanos ports: - - name: thanos-store + - name: store port: 10901 targetPort: 10901 ---- -apiVersion: v1 -kind: Service -metadata: - name: simplyblock-thanos-query - namespace: {{ .Release.Namespace }} -spec: - selector: - app: simplyblock-thanos-query - ports: - - name: thanos-query + - name: query port: 9091 targetPort: 9091 ---- -apiVersion: v1 -kind: Service -metadata: - name: simplyblock-node-exporter - namespace: {{ .Release.Namespace }} -spec: - selector: - app: simplyblock-node-exporter - ports: - - name: simplyblock-node-exporter - protocol: TCP - port: 9200 - targetPort: 9200 - --- apiVersion: v1 kind: Service diff --git a/simplyblock_core/scripts/charts/templates/simplyblock-manager.yaml b/simplyblock_core/scripts/charts/templates/simplyblock-manager.yaml new file mode 100644 index 000000000..257be5ae1 --- /dev/null +++ b/simplyblock_core/scripts/charts/templates/simplyblock-manager.yaml @@ -0,0 +1,196 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simplyblock-manager + labels: + control-plane: simplyblock-manager + app: simplyblock-manager +spec: + selector: + matchLabels: + app: simplyblock-manager + replicas: 1 + template: + metadata: + labels: + control-plane: simplyblock-manager + app: simplyblock-manager + spec: + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + serviceAccountName: simplyblock-manager + containers: + - image: simplyblock/simplyblock-manager:main + imagePullPolicy: Always + name: manager + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 500m + memory: 256Mi + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + privileged: false + terminationGracePeriodSeconds: 10 + +################# ROLE AND ROLE BINDING ############################## +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: simplyblock-manager + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: simplyblock-manager-clusterrole +rules: +- apiGroups: + - "" + resources: + - configmaps + - events + - persistentvolumeclaims + - pods + - pods/exec + - namespaces + - secrets + - services + - serviceaccounts + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + - daemonsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "rbac.authorization.k8s.io" + resources: + - roles + - clusterroles + verbs: + - create + - get + - list + - watch + - update + - patch +- apiGroups: + - "rbac.authorization.k8s.io" + resources: + - rolebindings + - clusterrolebindings + verbs: + - create + - get + - list + - watch + - update + - patch +- apiGroups: + - simplyblock.simplyblock.io + resources: + - simplyblockpools + - simplyblocklvols + - simplyblockstorageclusters + - simplyblockstoragenodes + - simplyblockdevices + - simplyblocktasks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - simplyblock.simplyblock.io + resources: + - simplyblockpools/finalizers + - simplyblocklvols/finalizers + - simplyblockstorageclusters/finalizers + - simplyblockstoragenodes/finalizers + - simplyblockdevices/finalizers + - simplyblocktasks/finalizers + verbs: + - update + - delete +- apiGroups: + - simplyblock.simplyblock.io + resources: + - simplyblockpools/status + - simplyblocklvols/status + - simplyblockstorageclusters/status + - simplyblockstoragenodes/status + - simplyblockdevices/status + - simplyblocktasks/status + verbs: + - get + - patch + - update + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: simplyblock-manager-clusterrolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: simplyblock-manager-clusterrole +subjects: +- kind: ServiceAccount + name: simplyblock-manager + namespace: {{ .Release.Namespace }} + \ No newline at end of file diff --git a/simplyblock_core/scripts/charts/templates/simplyblock_customresource.yaml b/simplyblock_core/scripts/charts/templates/simplyblock_customresource.yaml new file mode 100644 index 000000000..eb360b60a --- /dev/null +++ b/simplyblock_core/scripts/charts/templates/simplyblock_customresource.yaml @@ -0,0 +1,145 @@ +{{- if .Values.simplyblock.cluster }} +apiVersion: simplyblock.simplyblock.io/v1alpha1 +kind: SimplyBlockStorageCluster +metadata: + name: {{ .Values.simplyblock.cluster.clusterName }} + namespace: {{ .Release.Namespace }} +spec: + clusterName: {{ .Values.simplyblock.cluster.clusterName }} + + {{- if .Values.simplyblock.cluster.mgmtIfc }} + mgmtIfc: {{ .Values.simplyblock.cluster.mgmtIfc }} + {{- end }} + + {{- if .Values.simplyblock.cluster.fabric }} + fabric: {{ .Values.simplyblock.cluster.fabric }} + {{- end }} + + {{- if hasKey .Values.simplyblock.cluster "isSingleNode" }} + isSingleNode: {{ .Values.simplyblock.cluster.isSingleNode }} + {{- end }} + + {{- if hasKey .Values.simplyblock.cluster "enableNodeAffinity" }} + enableNodeAffinity: {{ .Values.simplyblock.cluster.enableNodeAffinity }} + {{- end }} + + {{- if hasKey .Values.simplyblock.cluster "strictNodeAntiAffinity" }} + strictNodeAntiAffinity: {{ .Values.simplyblock.cluster.strictNodeAntiAffinity }} + {{- end }} + + {{- if .Values.simplyblock.cluster.capWarn }} + capWarn: {{ .Values.simplyblock.cluster.capWarn }} + {{- end }} + + {{- if .Values.simplyblock.cluster.capCrit }} + capCrit: {{ .Values.simplyblock.cluster.capCrit }} + {{- end }} + + {{- if .Values.simplyblock.cluster.provCapWarn }} + provCapWarn: {{ .Values.simplyblock.cluster.provCapWarn }} + {{- end }} + + {{- if .Values.simplyblock.cluster.provCapCrit }} + provCapCrit: {{ .Values.simplyblock.cluster.provCapCrit }} + {{- end }} +{{- end }} + +--- +{{- if .Values.simplyblock.pool }} +apiVersion: simplyblock.simplyblock.io/v1alpha1 +kind: SimplyBlockPool +metadata: + name: {{ .Values.simplyblock.pool.name }} + namespace: {{ .Release.Namespace }} +spec: + name: {{ .Values.simplyblock.pool.name }} + clusterName: {{ .Values.simplyblock.cluster.clusterName }} + + {{- if .Values.simplyblock.pool.capacityLimit }} + capacityLimit: {{ .Values.simplyblock.pool.capacityLimit | quote }} + {{- end }} +{{- end }} + +--- +{{- if .Values.simplyblock.lvol }} +apiVersion: simplyblock.simplyblock.io/v1alpha1 +kind: SimplyBlockLvol +metadata: + name: {{ .Values.simplyblock.lvol.name }} + namespace: {{ .Release.Namespace }} +spec: + clusterName: {{ .Values.simplyblock.cluster.clusterName }} + poolName: {{ .Values.simplyblock.pool.name }} +{{- end }} + +--- +{{- if .Values.simplyblock.storageNodes }} +apiVersion: simplyblock.simplyblock.io/v1alpha1 +kind: SimplyBlockStorageNode +metadata: + name: {{ .Values.simplyblock.storageNodes.name }} + namespace: {{ .Release.Namespace }} +spec: + clusterName: {{ .Values.simplyblock.cluster.clusterName }} + + {{- if .Values.simplyblock.storageNodes.clusterImage }} + clusterImage: {{ .Values.simplyblock.storageNodes.clusterImage }} + {{- end }} + + {{- if .Values.simplyblock.storageNodes.mgmtIfc }} + mgmtIfc: {{ .Values.simplyblock.storageNodes.mgmtIfc }} + {{- end }} + + {{- if .Values.simplyblock.storageNodes.maxLVol }} + maxLVol: {{ .Values.simplyblock.storageNodes.maxLVol }} + {{- end }} + + {{- if .Values.simplyblock.storageNodes.maxSize }} + maxSize: {{ .Values.simplyblock.storageNodes.maxSize | quote }} + {{- end }} + + {{- if hasKey .Values.simplyblock.storageNodes "partitions" }} + partitions: {{ .Values.simplyblock.storageNodes.partitions }} + {{- end }} + + {{- if .Values.simplyblock.storageNodes.corePercentage }} + corePercentage: {{ .Values.simplyblock.storageNodes.corePercentage }} + {{- end }} + + {{- if hasKey .Values.simplyblock.storageNodes "spdkDebug" }} + spdkDebug: {{ .Values.simplyblock.storageNodes.spdkDebug }} + {{- end }} + + {{- if hasKey .Values.simplyblock.storageNodes "coreIsolation" }} + coreIsolation: {{ .Values.simplyblock.storageNodes.coreIsolation }} + {{- end }} + + {{- if .Values.simplyblock.storageNodes.workerNodes }} + workerNodes: + {{- range .Values.simplyblock.storageNodes.workerNodes }} + - {{ . }} + {{- end }} + {{- end }} +{{- end }} + +--- +{{- if .Values.simplyblock.devices }} +apiVersion: simplyblock.simplyblock.io/v1alpha1 +kind: SimplyBlockDevice +metadata: + name: {{ .Values.simplyblock.devices.name }} + namespace: {{ .Release.Namespace }} +spec: + clusterName: {{ .Values.simplyblock.cluster.clusterName }} +{{- end }} + +--- +{{- if .Values.simplyblock.tasks }} +apiVersion: simplyblock.simplyblock.io/v1alpha1 +kind: SimplyBlockTask +metadata: + name: {{ .Values.simplyblock.tasks.name }} + namespace: {{ .Release.Namespace }} +spec: + clusterName: {{ .Values.simplyblock.cluster.clusterName }} +{{- end }} diff --git a/simplyblock_core/scripts/charts/values.yaml b/simplyblock_core/scripts/charts/values.yaml index 0b70f321e..7272846d7 100644 --- a/simplyblock_core/scripts/charts/values.yaml +++ b/simplyblock_core/scripts/charts/values.yaml @@ -1,29 +1,26 @@ -graylog: - rootPasswordSha2: "b87c15a8ae4736d771ca60a7cc2014baaeab19b11c31f5fedef9421958a403c9" - passwordSecret: "is6SP2EdWg0NdmVGv6CEp5hRHNL7BKVMFem4t9pouMqDQnHwXMSomas1qcbKSt5yISr8eBHv4Y7Dbswhyz84Ut0TW6kqsiPs" -monitoring: - enabled: true +observability: + enabled: false secret: "sWbpOgba1bKnCfcPkVQi" - -log: deletionInterval: "3d" - retentionPeriod: "7d" level: "DEBUG" - maxNumberIndex: "3" - -grafana: - endpoint: "" - contactPoint: "https://hooks.slack.com/services/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" + graylog: + rootPasswordSha2: "b87c15a8ae4736d771ca60a7cc2014baaeab19b11c31f5fedef9421958a403c9" + passwordSecret: "is6SP2EdWg0NdmVGv6CEp5hRHNL7BKVMFem4t9pouMqDQnHwXMSomas1qcbKSt5yISr8eBHv4Y7Dbswhyz84Ut0TW6kqsiPs" + maxNumberIndex: "3" + retentionPeriod: "7d" + grafana: + endpoint: "" + contactPoint: "https://hooks.slack.com/services/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" image: simplyblock: repository: "public.ecr.aws/simply-block/simplyblock" - tag: "main" + tag: "improved_user_experience" pullPolicy: "Always" ports: - lvolNvmfPortStart: + lvolNvmfPortStart: 9100 storageclass: allowedTopologyZones: [] @@ -61,7 +58,7 @@ opensearch: persistence: enabled: true storageClass: local-hostpath - size: 10Gi + size: 20Gi resources: requests: @@ -180,7 +177,7 @@ prometheus: enabled: false ingress: - enabled: true + enabled: false ingressClassName: nginx useDNS: false host: "" @@ -193,3 +190,45 @@ ingress: extraArgs: tcp-services-configmap: "simplyblock/simplyblock-tcp-services" nodeSelector: {} + + +simplyblock: + cluster: + clusterName: demo-cluster + mgmtIfc: eth0 + fabric: tcp + isSingleNode: false + enableNodeAffinity: false + strictNodeAntiAffinity: false + capWarn: 80 + capCrit: 90 + provCapWarn: 120 + provCapCrit: 150 + + pool: + name: demo-pool + capacityLimit: 100Gi + + lvol: + name: demo-lvol + + storageNodes: + name: demo-node + clusterImage: public.ecr.aws/simply-block/simplyblock:improved_user_experience + mgmtIfc: eth0 + maxLVol: 10 + maxSize: 150G + partitions: 0 + corePercentage: 65 + spdkDebug: false + coreIsolation: false + workerNodes: + - vm02.simplyblock3.localdomain + - vm03.simplyblock3.localdomain + - vm04.simplyblock3.localdomain + + devices: + name: demo-devices + + tasks: + name: demo-task diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index a6d89b74d..1e14d7a80 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -835,8 +835,8 @@ def add_node(cluster_id, node_addr, iface_name,data_nics_list, max_snap, spdk_image=None, spdk_debug=False, small_bufsize=0, large_bufsize=0, num_partitions_per_dev=0, jm_percent=0, enable_test_device=False, - namespace=None, enable_ha_jm=False, id_device_by_nqn=False, - partition_size="", ha_jm_count=3): + namespace=None, enable_ha_jm=False, cr_name=None, cr_namespace=None, cr_plural=None, + id_device_by_nqn=False, partition_size="", ha_jm_count=3): snode_api = SNodeClient(node_addr) node_info, _ = snode_api.info() if node_info.get("nodes_config") and node_info["nodes_config"].get("nodes"): @@ -1086,6 +1086,9 @@ def add_node(cluster_id, node_addr, iface_name,data_nics_list, snode.cloud_name = cloud_instance['cloud'] or "" snode.namespace = namespace + snode.cr_name = cr_name + snode.cr_namespace = cr_namespace + snode.cr_plural = cr_plural snode.ssd_pcie = ssd_pcie snode.hostname = hostname snode.host_nqn = subsystem_nqn @@ -1148,7 +1151,7 @@ def add_node(cluster_id, node_addr, iface_name,data_nics_list, snode.physical_label = 0 else: snode.physical_label = get_next_physical_device_order(snode) - + snode.num_partitions_per_dev = num_partitions_per_dev snode.jm_percent = jm_percent snode.id_device_by_nqn = id_device_by_nqn @@ -1683,7 +1686,7 @@ def restart_storage_node( cluster_ip = cluster_docker.info()["Swarm"]["NodeAddr"] else: - cluster_ip = utils.get_k8s_node_ip() + cluster_ip = utils.get_k8s_node_ip() total_mem = 0 for n in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id): @@ -1864,7 +1867,7 @@ def restart_storage_node( snode.nvme_devices.append(dev) snode.write_to_db(db_controller.kv_store) - if node_ip and len(new_devices)>0: + if node_ip: # prepare devices on new node if snode.num_partitions_per_dev == 0 or snode.jm_percent == 0: diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 1f086dc2d..6711dc857 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -10,8 +10,8 @@ import sys import uuid import time -import socket -from typing import Union, Any, Optional, Tuple +from typing import Union, Any, Optional, Tuple, Dict +from datetime import datetime, timezone from docker import DockerClient from kubernetes import client, config from kubernetes.client import ApiException, V1Deployment, V1DeploymentSpec, V1ObjectMeta, \ @@ -198,16 +198,8 @@ def get_k8s_node_ip(): logger.error("No mgmt nodes was found in the cluster!") return False - mgmt_ips = [node.mgmt_ip for node in nodes] - - for ip in mgmt_ips: - try: - with socket.create_connection((ip, 10250), timeout=2): - return ip - except Exception as e: - print(e) - raise e - return False + for node in nodes: + return node.mgmt_ip def dict_agg(data, mean=False, keys=None): @@ -1937,6 +1929,254 @@ def load_kube_config_with_fallback(): except Exception: config.load_kube_config() +def patch_cr_status( + *, + group: str, + version: str, + plural: str, + namespace: str, + name: str, + status_patch: dict, +): + """ + Patch the status subresource of a Custom Resource. + + status_patch example: + {"": "": } + """ + + load_kube_config_with_fallback() + + api = client.CustomObjectsApi() + + body = { + "status": status_patch + } + + try: + api.patch_namespaced_custom_object_status( + group=group, + version=version, + namespace=namespace, + plural=plural, + name=name, + body=body, + ) + except ApiException as e: + raise RuntimeError( + f"Failed to patch status for {name}: {e.reason} {e.body}" + ) + +def patch_cr_node_status( + *, + group: str, + version: str, + plural: str, + namespace: str, + name: str, + node_uuid: str, + node_mgmt_ip: str, + updates: Optional[Dict[str, Any]] = None, + remove: bool = False, +): + """ + Patch status.nodes[*] fields for a specific node identified by UUID. + + Operations: + - Update a node (by uuid or mgmtIp) + - Remove a node (by uuid or mgmtIp) + + updates example: + {"health": "true"} + {"status": "offline"} + {"capacity": {"sizeUsed": 1234}} + """ + load_kube_config_with_fallback() + api = client.CustomObjectsApi() + + try: + cr = api.get_namespaced_custom_object( + group=group, + version=version, + namespace=namespace, + plural=plural, + name=name, + ) + + status_nodes = cr.get("status", {}).get("nodes", []) + if not status_nodes: + raise RuntimeError("CR has no status.nodes") + + spec_worker_nodes = cr.get("spec", {}).get("workerNodes", []) + + found = False + new_status_nodes = [] + removed_hostname = None + + for node in status_nodes: + match = ( + node.get("uuid") == node_uuid or + node.get("mgmtIp") == node_mgmt_ip + ) + + if match: + found = True + removed_hostname = node.get("hostname") + + if remove: + continue + + if updates: + node.update(updates) + + new_status_nodes.append(node) + + if not found: + raise RuntimeError( + f"Node not found (uuid={node_uuid}, mgmtIp={node_mgmt_ip})" + ) + + if remove and removed_hostname: + new_worker_nodes = [ + n for n in spec_worker_nodes if n != removed_hostname + ] + + api.patch_namespaced_custom_object( + group=group, + version=version, + namespace=namespace, + plural=plural, + name=name, + body={ + "spec": { + "workerNodes": new_worker_nodes + } + }, + ) + + api.patch_namespaced_custom_object_status( + group=group, + version=version, + namespace=namespace, + plural=plural, + name=name, + body={ + "status": { + "nodes": new_status_nodes + } + }, + ) + + except ApiException as e: + raise RuntimeError( + f"Failed to patch node for {name}: {e.reason} {e.body}" + ) + +def patch_cr_lvol_status( + *, + group: str, + version: str, + plural: str, + namespace: str, + name: str, + lvol_uuid: Optional[str] = None, + updates: Optional[Dict[str, Any]] = None, + remove: bool = False, + add: Optional[Dict[str, Any]] = None, +): + """ + Patch status.lvols[*] for an LVOL CustomResource. + + Operations: + - Update an existing LVOL (by uuid) + - Remove an LVOL (by uuid) + - Add a new LVOL entry + + Parameters: + lvol_uuid: + UUID of the lvol entry to update or remove + + updates: + Dict of fields to update on the matched lvol + Example: + {"status": "offline", "health": False} + + remove: + If True, remove the lvol identified by lvol_uuid + + add: + Full lvol dict to append to status.lvols + """ + + load_kube_config_with_fallback() + api = client.CustomObjectsApi() + + now = datetime.now(timezone.utc).isoformat() + + try: + cr = api.get_namespaced_custom_object( + group=group, + version=version, + namespace=namespace, + plural=plural, + name=name, + ) + + status = cr.get("status", {}) + lvols = status.get("lvols", []) + + # Ensure list exists + if lvols is None: + lvols = [] + + # ---- ADD ---- + if add is not None: + add.setdefault("createDt", now) + add["updateDt"] = now + lvols.append(add) + + # ---- UPDATE / REMOVE ---- + if lvol_uuid: + found = False + new_lvols = [] + + for lvol in lvols: + if lvol.get("uuid") == lvol_uuid: + found = True + + if remove: + continue + + if updates: + lvol.update(updates) + lvol["updateDt"] = now + + new_lvols.append(lvol) + + if not found: + raise RuntimeError(f"LVOL not found (uuid={lvol_uuid})") + + lvols = new_lvols + + body = { + "status": { + "lvols": lvols + } + } + + api.patch_namespaced_custom_object_status( + group=group, + version=version, + namespace=namespace, + plural=plural, + name=name, + body=body, + ) + + except ApiException as e: + raise RuntimeError( + f"Failed to patch lvol status for {name}: {e.reason} {e.body}" + ) def get_node_name_by_ip(target_ip: str) -> str: load_kube_config_with_fallback() diff --git a/simplyblock_web/api/v1/__init__.py b/simplyblock_web/api/v1/__init__.py index 084a737cc..6df2a2db5 100644 --- a/simplyblock_web/api/v1/__init__.py +++ b/simplyblock_web/api/v1/__init__.py @@ -1,5 +1,5 @@ import logging -import os +import fdb from flask import jsonify from flask import Flask @@ -45,28 +45,21 @@ def status(): @api.route('/health/fdb', methods=['GET']) def health_fdb(): - fdb_cluster_file = constants.KVD_DB_FILE_PATH + try: + fdb.api_version(constants.KVD_DB_VERSION) + + db = fdb.open(constants.KVD_DB_FILE_PATH) + tr = db.create_transaction() + + tr.get(b"\x00") + tr.commit().wait() - if not os.path.exists(fdb_cluster_file): return jsonify({ - "fdb_connected": False, - "message": "FDB cluster file not found" - }), 503 + "fdb_connected": True + }), 200 - try: - with open(fdb_cluster_file, 'r') as f: - cluster_data = f.read().strip() - if not cluster_data: - return jsonify({ - "fdb_connected": False, - "message": "FDB cluster file is empty" - }), 503 except Exception as e: return jsonify({ "fdb_connected": False, - "message": f"Failed to read FDB cluster file: {str(e)}" + "error": str(e) }), 503 - - return jsonify({ - "fdb_connected": True, - }), 200 diff --git a/simplyblock_web/api/v1/cluster.py b/simplyblock_web/api/v1/cluster.py index 2447cf958..9bb191602 100644 --- a/simplyblock_web/api/v1/cluster.py +++ b/simplyblock_web/api/v1/cluster.py @@ -47,6 +47,9 @@ def add_cluster(): qpair_count = cl_data.get('qpair_count', 256) name = cl_data.get('name', None) fabric = cl_data.get('fabric', "tcp") + cr_name = cl_data.get('cr_name', None) + cr_namespace = cl_data.get('cr_namespace', None) + cr_plural = cl_data.get('cr_plural', None) max_queue_size = cl_data.get('max_queue_size', 128) inflight_io_threshold = cl_data.get('inflight_io_threshold', 4) @@ -56,7 +59,8 @@ def add_cluster(): return utils.get_response(cluster_ops.add_cluster( blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, enable_node_affinity, - qpair_count, max_queue_size, inflight_io_threshold, strict_node_anti_affinity, is_single_node, name, fabric + qpair_count, max_queue_size, inflight_io_threshold, strict_node_anti_affinity, is_single_node, name, + cr_name, cr_namespace, cr_plural, fabric )) @@ -78,7 +82,7 @@ def create_first_cluster(): distr_npcs = cl_data.get('distr_npcs', 1) distr_bs = cl_data.get('distr_bs', 4096) distr_chunk_bs = cl_data.get('distr_chunk_bs', 4096) - ha_type = cl_data.get('ha_type', 'single') + ha_type = cl_data.get('ha_type', 'ha') enable_node_affinity = cl_data.get('enable_node_affinity', False) qpair_count = cl_data.get('qpair_count', 256) name = cl_data.get('name', None) @@ -91,6 +95,9 @@ def create_first_cluster(): inflight_io_threshold = cl_data.get('inflight_io_threshold', 4) strict_node_anti_affinity = cl_data.get('strict_node_anti_affinity', False) is_single_node = cl_data.get('is_single_node', False) + cr_name = cl_data.get('cr_name', None) + cr_namespace = cl_data.get('cr_namespace', None) + cr_plural = cl_data.get('cr_plural', None) cluster_ip = cl_data.get('cluster_ip', None) grafana_secret = cl_data.get('grafana_secret', None) @@ -98,8 +105,8 @@ def create_first_cluster(): cluster_id = cluster_ops.add_cluster( blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, enable_node_affinity, - qpair_count, max_queue_size, inflight_io_threshold, strict_node_anti_affinity, is_single_node, name, fabric, - cluster_ip=cluster_ip, grafana_secret=grafana_secret) + qpair_count, max_queue_size, inflight_io_threshold, strict_node_anti_affinity, is_single_node, name, + cr_name, cr_namespace, cr_plural, fabric, cluster_ip=cluster_ip, grafana_secret=grafana_secret) if cluster_id: return utils.get_response(db.get_cluster_by_id(cluster_id).to_dict()) else: diff --git a/simplyblock_web/api/v2/__init__.py b/simplyblock_web/api/v2/__init__.py index c3723cce6..c4c0168c7 100644 --- a/simplyblock_web/api/v2/__init__.py +++ b/simplyblock_web/api/v2/__init__.py @@ -38,6 +38,9 @@ def _verify_api_token( storage_node.api.include_router(storage_node.instance_api) cluster.instance_api.include_router(storage_node.api) + +task.api.include_router(task.instance_api) + cluster.instance_api.include_router(task.api) volume.api.include_router(volume.instance_api) diff --git a/simplyblock_web/api/v2/cluster.py b/simplyblock_web/api/v2/cluster.py index 49f8a09e8..009e4cd96 100644 --- a/simplyblock_web/api/v2/cluster.py +++ b/simplyblock_web/api/v2/cluster.py @@ -35,7 +35,7 @@ class ClusterParams(BaseModel): distr_npcs: int = 1 distr_bs: int = 4096 distr_chunk_bs: int = 4096 - ha_type: Literal['single', 'ha'] = 'single' + ha_type: Literal['single', 'ha'] = 'ha' qpair_count: int = 256 max_queue_size: int = 128 inflight_io_threshold: int = 4 @@ -43,6 +43,9 @@ class ClusterParams(BaseModel): strict_node_anti_affinity: bool = False is_single_node: bool = False fabric: str = "tcp" + cr_name: str = "" + cr_namespace: str = "" + cr_plural: str = "" cluster_ip: str = "" grafana_secret: str = "" @@ -163,6 +166,13 @@ def activate(cluster: Cluster) -> Response: ).start() return Response(status_code=202) # FIXME: Provide URL for checking task status +@instance_api.post('/expand', name='clusters:expand', status_code=202, responses={202: {"content": None}}) +def expand(cluster: Cluster) -> Response: + Thread( + target=cluster_ops.cluster_expand, + args=(cluster.get_id(),), + ).start() + return Response(status_code=202) # FIXME: Provide URL for checking task status @instance_api.post('/update', name='clusters:upgrade', status_code=204, responses={204: {"content": None}}) def update_cluster( cluster: Cluster, parameters: _UpdateParams) -> Response: diff --git a/simplyblock_web/api/v2/device.py b/simplyblock_web/api/v2/device.py index 4fa0949fb..b0015b69b 100644 --- a/simplyblock_web/api/v2/device.py +++ b/simplyblock_web/api/v2/device.py @@ -49,13 +49,19 @@ def get(cluster: Cluster, storage_node: StorageNode, device: Device) -> DeviceDT return DeviceDTO.from_model(device, stat_obj) -@instance_api.delete('/', name='clusters:storage_nodes:devices:delete', status_code=204, responses={204: {"content": None}}) -def delete(cluster: Cluster, storage_node: StorageNode, device: Device) -> Response: - if not device_controller.device_remove(device.get_id()): +@instance_api.post('/remove', name='clusters:storage_nodes:devices:remove', status_code=204, responses={204: {"content": None}}) +def remove(cluster: Cluster, storage_node: StorageNode, device: Device, force: bool = False) -> Response: + if not device_controller.device_remove(device.get_id(), force): raise ValueError('Failed to remove device') return Response(status_code=204) +@instance_api.post('/restart', name='clusters:storage_nodes:devices:restart', status_code=204, responses={204: {"content": None}}) +def restart(cluster: Cluster, storage_node: StorageNode, device: Device, force: bool = False) -> Response: + if not device_controller.restart_device(device.get_id(), force): + raise ValueError('Failed to restart device') + + return Response(status_code=204) @instance_api.get('/capacity', name='clusters:storage_nodes:devices:capacity') def capacity( diff --git a/simplyblock_web/api/v2/dtos.py b/simplyblock_web/api/v2/dtos.py index 62f1a94e1..b30e3ee26 100644 --- a/simplyblock_web/api/v2/dtos.py +++ b/simplyblock_web/api/v2/dtos.py @@ -40,13 +40,12 @@ def from_model(model: StatsObject): class ClusterDTO(BaseModel): - id: UUID + uuid: UUID name: Optional[str] nqn: str status: Literal['active', 'read_only', 'inactive', 'suspended', 'degraded', 'unready', 'in_activation', 'in_expansion'] - rebalancing: bool + is_re_balancing: bool block_size: util.Unsigned - coding: Tuple[util.Unsigned, util.Unsigned] ha: bool utliziation_critical: util.Percent utilization_warning: util.Percent @@ -55,18 +54,21 @@ class ClusterDTO(BaseModel): node_affinity: bool anti_affinity: bool secret: str + distr_ndcs: int + distr_npcs: int capacity: CapacityStatDTO @staticmethod def from_model(model: Cluster, stat_obj: Optional[StatsObject]=None): return ClusterDTO( - id=UUID(model.get_id()), + uuid=UUID(model.get_id()), name=model.cluster_name, nqn=model.nqn, status=model.status, # type: ignore - rebalancing=model.is_re_balancing, + is_re_balancing=model.is_re_balancing, block_size=model.blk_size, - coding=(model.distr_ndcs, model.distr_npcs), + distr_ndcs=model.distr_ndcs, + distr_npcs=model.distr_npcs, ha=model.ha_type == 'ha', utilization_warning=model.cap_warn, utliziation_critical=model.cap_crit, @@ -181,6 +183,13 @@ def from_model(model: SnapShot, request: Request, cluster_id, pool_id, volume_id class StorageNodeDTO(BaseModel): uuid: UUID status: str + hostname: str + cpu: int + spdk_mem: int + lvols: int + rpc_port: int + lvol_subsys_port: int + nvmf_port: int mgmt_ip: IPv4Address health_check: bool online_devices: str @@ -191,6 +200,13 @@ def from_model(model: StorageNode, stat_obj: Optional[StatsObject]=None): return StorageNodeDTO( uuid=UUID(model.get_id()), status=model.status, + hostname=model.hostname, + cpu=model.cpu, + spdk_mem=model.spdk_mem, + lvols=model.lvols, + rpc_port=model.rpc_port, + lvol_subsys_port=model.lvol_subsys_port, + nvmf_port=model.nvmf_port, mgmt_ip=IPv4Address(model.mgmt_ip), health_check=model.health_check, online_devices=f"{len(model.nvme_devices)}/{len([d for d in model.nvme_devices if d.status=='online'])}", @@ -211,7 +227,7 @@ class TaskDTO(BaseModel): @staticmethod def from_model(model: JobSchedule): return TaskDTO( - id=UUID(model.get_id()), + id=UUID(model.uuid), status=model.status, canceled=model.canceled, function_name=model.function_name, @@ -228,12 +244,24 @@ class VolumeDTO(BaseModel): status: str health_check: bool nqn: str + hostname: str + fabric: str nodes: List[util.UrlPath] port: util.Port size: util.Unsigned + ndcs: int + npcs: int + pool_uuid: str + pool_name: str + pvc_name: str = "" + snapshot_name: str = "" + blobid: int + ns_id: int cloned_from: Optional[util.UrlPath] crypto_key: Optional[Tuple[str, str]] high_availability: bool + lvol_priority_class: util.Unsigned + max_namespace_per_subsys: int max_rw_iops: util.Unsigned max_rw_mbytes: util.Unsigned max_r_mbytes: util.Unsigned @@ -248,6 +276,8 @@ def from_model(model: LVol, request: Request, cluster_id: str, stat_obj: Optiona status=model.status, health_check=model.health_check, nqn=model.nqn, + hostname=model.hostname, + fabric=model.fabric, nodes=[ str(request.url_for( 'clusters:storage-nodes:detail', @@ -270,6 +300,16 @@ def from_model(model: LVol, request: Request, cluster_id: str, stat_obj: Optiona else None ), high_availability=model.ha_type == 'ha', + pool_uuid=model.pool_uuid, + pool_name=model.pool_name, + pvc_name=model.pvc_name, + snapshot_name=model.snapshot_name, + ndcs=model.ndcs, + npcs=model.npcs, + blobid=model.blobid, + ns_id=model.ns_id, + lvol_priority_class=model.lvol_priority_class, + max_namespace_per_subsys=model.max_namespace_per_subsys, max_rw_iops=model.rw_ios_per_sec, max_rw_mbytes=model.rw_mbytes_per_sec, max_r_mbytes=model.r_mbytes_per_sec, diff --git a/simplyblock_web/api/v2/pool.py b/simplyblock_web/api/v2/pool.py index 4ccae01ab..4bc201a28 100644 --- a/simplyblock_web/api/v2/pool.py +++ b/simplyblock_web/api/v2/pool.py @@ -39,6 +39,9 @@ class StoragePoolParams(BaseModel): max_rw_mbytes: util.Unsigned = 0 max_r_mbytes: util.Unsigned = 0 max_w_mbytes: util.Unsigned = 0 + cr_name: str + cr_namespace: str + cr_plural: str @api.post('/', name='clusters:storage-pools:create', status_code=201, responses={201: {"content": None}}) @@ -52,7 +55,7 @@ def add(request: Request, cluster: Cluster, parameters: StoragePoolParams) -> Re id_or_false = pool_controller.add_pool( parameters.name, parameters.pool_max, parameters.volume_max_size, parameters.max_rw_iops, parameters.max_rw_mbytes, - parameters.max_r_mbytes, parameters.max_w_mbytes, cluster.get_id() + parameters.max_r_mbytes, parameters.max_w_mbytes, cluster.get_id(), parameters.cr_name, parameters.cr_namespace, parameters.cr_plural ) if not id_or_false: @@ -102,6 +105,9 @@ class UpdatableStoragePoolParams(BaseModel): max_rw_mbytes: Optional[util.Unsigned] = None max_r_mbytes: Optional[util.Unsigned] = None max_w_mbytes: Optional[util.Unsigned] = None + lvols_cr_name: Optional[str] = None + lvols_cr_namespace: Optional[str] = None + lvols_cr_plural: Optional[str] = None @instance_api.put('/', name='clusters:storage-pools:update', status_code=204, responses={204: {"content": None}}) diff --git a/simplyblock_web/api/v2/storage_node.py b/simplyblock_web/api/v2/storage_node.py index aa7923d36..5159425f1 100644 --- a/simplyblock_web/api/v2/storage_node.py +++ b/simplyblock_web/api/v2/storage_node.py @@ -35,17 +35,21 @@ def list(cluster: Cluster) -> List[StorageNodeDTO]: class StorageNodeParams(BaseModel): node_address: Annotated[str, Field(web_utils.IP_PATTERN)] interface_name: str - max_snapshots: int = Field(500) - ha_jm: bool = Field(True) - test_device: bool = Field(False) + max_snapshots: Optional[int] = Field(500) + ha_jm: Optional[bool] = Field(True) + test_device: Optional[bool] = Field(False) spdk_image: Optional[str] = Field("") spdk_debug: bool = Field(False) data_nics: List[str] = Field([]) namespace: str = Field('default') + id_device_by_nqn: Optional[bool] = Field(False) jm_percent: util.Percent = Field(3) partitions: int = Field(1) iobuf_small_pool_count: int = Field(0) iobuf_large_pool_count: int = Field(0) + cr_name: str + cr_namespace: str + cr_plural: str @api.post('/', name='clusters:storage-nodes:create', status_code=201, responses={201: {"content": None}}) @@ -67,6 +71,10 @@ def add(request: Request, cluster: Cluster, parameters: StorageNodeParams): 'enable_test_device': parameters.test_device, 'namespace': parameters.namespace, 'enable_ha_jm': parameters.ha_jm, + 'id_device_by_nqn': parameters.id_device_by_nqn, + 'cr_name': parameters.cr_name, + 'cr_namespace': parameters.cr_namespace, + 'cr_plural': parameters.cr_plural, } ) if not task_id_or_false: @@ -98,12 +106,19 @@ def get(cluster: Cluster, storage_node: StorageNode): @instance_api.delete('/', name='clusters:storage-nodes:delete') def delete( - cluster: Cluster, storage_node: StorageNode, force_remove: bool = False, force_migrate: bool = False) -> Response: + cluster: Cluster, storage_node: StorageNode, force_remove: bool = False, force_migrate: bool = False, force_delete: bool = False ) -> Response: none_or_false = storage_node_ops.remove_storage_node( storage_node.get_id(), force_remove=force_remove, force_migrate=force_migrate ) if none_or_false == False: # noqa raise ValueError('Failed to remove storage node') + + if force_delete: + none_or_false = storage_node_ops.delete_storage_node( + storage_node.get_id(), force=force_delete + ) + if none_or_false == False: # noqa + raise ValueError('Failed to delete storage node') return Response(status_code=204) @@ -201,17 +216,20 @@ def shutdown(cluster: Cluster, storage_node: StorageNode, force: bool = False) - class _RestartParams(BaseModel): force: bool = False reattach_volume: bool = False + node_address: Optional[Annotated[str, Field(pattern=web_utils.IP_PATTERN)]] = None + @instance_api.post('/start', name='clusters:storage-nodes:start', status_code=202, responses={202: {"content": None}}) # Same as restart for now @instance_api.post('/restart', name='clusters:storage-nodes:restart', status_code=202, responses={202: {"content": None}}) -def restart(cluster: Cluster, storage_node: StorageNode, parameters: _RestartParams = _RestartParams()) -> Response: +def restart(cluster: Cluster, storage_node: StorageNode, parameters: _RestartParams) -> Response: storage_node = storage_node Thread( target=storage_node_ops.restart_storage_node, kwargs={ "node_id": storage_node.get_id(), "force": parameters.force, + "node_ip": parameters.node_address, "reattach_volume": parameters.reattach_volume, } ).start() diff --git a/simplyblock_web/api/v2/task.py b/simplyblock_web/api/v2/task.py index 83890640f..94ecccce3 100644 --- a/simplyblock_web/api/v2/task.py +++ b/simplyblock_web/api/v2/task.py @@ -5,7 +5,6 @@ from simplyblock_core.db_controller import DBController from simplyblock_core.models.job_schedule import JobSchedule -from simplyblock_core.controllers import tasks_controller from .cluster import Cluster from .dtos import TaskDTO @@ -16,13 +15,13 @@ @api.get('/', name='clusters:tasks:list') def list(cluster: Cluster) -> List[TaskDTO]: - return [ - TaskDTO.from_model(task) - for task - in tasks_controller.list_tasks(cluster.get_id()) - if task.cluster_id == cluster.get_id() - ] - + cluster_tasks = db.get_job_tasks(cluster.get_id(), limit=0) + data=[] + for t in cluster_tasks: + if t.function_name == JobSchedule.FN_DEV_MIG: + continue + data.append(t) + return [TaskDTO.from_model(task) for task in data] instance_api = APIRouter(prefix='/{task_id}') @@ -40,5 +39,3 @@ def _lookup_task(task_id: UUID) -> JobSchedule: @instance_api.get('/', name='clusters:tasks:detail') def get(cluster: Cluster, task: Task) -> TaskDTO: return TaskDTO.from_model(task) - -api.include_router(instance_api) diff --git a/simplyblock_web/templates/storage_core_isolation.yaml.j2 b/simplyblock_web/templates/storage_core_isolation.yaml.j2 index b6fafe2ee..9ae4ba8e8 100644 --- a/simplyblock_web/templates/storage_core_isolation.yaml.j2 +++ b/simplyblock_web/templates/storage_core_isolation.yaml.j2 @@ -91,7 +91,7 @@ spec: - name: etc mountPath: /etc - name: rootfs - mountPath: / + mountPath: /host - name: var-simplyblock mountPath: /var/simplyblock command: ["/bin/sh", "-c"] @@ -113,13 +113,13 @@ spec: apt update && apt install -y grep jq nvme-cli tuned ;; ubuntu) - apt update && apt install -y grep jq nvme-cli tuned - apt-get install -y linux-modules-extra-$(uname -r) + chroot /host apt update && chroot /host apt install -y grep jq nvme-cli tuned + chroot /host apt-get install -y linux-modules-extra-$(uname -r) ;; centos|rhel|rocky|almalinux) export YUM_RELEASEVER=$(awk -F'=' '/^VERSION_ID=/{gsub(/"/,"",$2); print $2}' /etc/os-release) export DNF_RELEASEVER=$(awk -F'=' '/^VERSION_ID=/{gsub(/"/,"",$2); print $2}' /etc/os-release) - dnf install -y grep jq nvme-cli kernel-modules-extra tuned \ + chroot /host dnf install -y grep jq nvme-cli kernel-modules-extra tuned \ --setopt=tsflags=nocontexts,noscripts --setopt=install_weak_deps=False ;; *) @@ -204,7 +204,7 @@ spec: tuned-adm profile "$TUNED_PROFILE" case "$OS_ID" in centos|rhel|rocky|almalinux) - grubby --update-kernel=ALL --args="isolcpus=$ISOLATED_CORES nohz_full=$ISOLATED_CORES rcu_nocbs=$ISOLATED_CORES" + chroot /host grubby --update-kernel=ALL --args="isolcpus=$ISOLATED_CORES nohz_full=$ISOLATED_CORES rcu_nocbs=$ISOLATED_CORES" ;; *) echo "" diff --git a/simplyblock_web/templates/storage_deploy_spdk.yaml.j2 b/simplyblock_web/templates/storage_deploy_spdk.yaml.j2 index 81f1e1eda..d118927cd 100644 --- a/simplyblock_web/templates/storage_deploy_spdk.yaml.j2 +++ b/simplyblock_web/templates/storage_deploy_spdk.yaml.j2 @@ -16,6 +16,7 @@ spec: nodeSelector: kubernetes.io/hostname: {{ HOSTNAME }} hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet tolerations: - effect: NoSchedule operator: Exists @@ -57,6 +58,11 @@ spec: - name: dockercontainerlogdirectory hostPath: path: /var/log/pods + {% if MODE == "kubernetes" %} + - name: config + configMap: + name: simplyblock-fluent-bit-config + {% endif %} initContainers: - name: copy-script @@ -160,4 +166,19 @@ spec: - name: dockercontainerlogdirectory mountPath: /var/log/pods readOnly: true + {% elif MODE == "kubernetes" %} + - name: fluent-bit + image: fluent/fluent-bit:1.8.11 + volumeMounts: + - name: varlog + mountPath: /var/log + - name: config + mountPath: /fluent-bit/etc/ + resources: + requests: + cpu: "100m" + memory: "200Mi" + limits: + cpu: "400m" + memory: "2Gi" {% endif %}