{ "__inputs": [ { "name": "DS_Metrics", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" }, { "name": "DS_Logs", "label": "VictoriaLogs", "description": "", "type": "datasource", "pluginId": "victoriametrics-logs-datasource", "pluginName": "VictoriaLogs" } ], "title": "Exordos Compute", "uid": "exordos-compute", "description": "Nodes, node sets and machine pools of Exordos: what they are, their status over time, how often they are created and deleted, pool capacity and node logs. Fed by the core state metrics (exordos_compute_*), which exordos_core writes every 15s for the node_exporter textfile collector.", "tags": [ "exordos", "compute" ], "editable": true, "graphTooltip": 1, "schemaVersion": 39, "version": 1, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "timezone": "", "links": [ { "title": "Exordos Elements", "type": "link", "url": "/d/exordos-elements/exordos-elements", "icon": "dashboard", "keepTime": true, "includeVars": false, "targetBlank": false } ], "annotations": { "list": [] }, "templating": { "list": [ { "name": "DS_Metrics", "label": "Metrics", "type": "datasource", "query": "prometheus", "current": {}, "hide": 0, "includeAll": false, "multi": false, "options": [], "refresh": 1, "regex": "" }, { "name": "DS_Logs", "label": "Logs", "type": "datasource", "query": "victoriametrics-logs-datasource", "current": {}, "hide": 0, "includeAll": false, "multi": false, "options": [], "refresh": 1, "regex": "" }, { "name": "element", "label": "Element", "type": "query", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "definition": "label_values(exordos_compute_node_status, element)", "query": { "qryType": 1, "query": "label_values(exordos_compute_node_status, element)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "multi": true, "includeAll": true, "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "sort": 1 }, { "name": "host", "label": "Host", "type": "query", "description": "Node whose logs the Node logs panel shows.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "definition": "label_values(exordos_compute_node_info{element=~\"$element\"}, hostname)", "query": { "qryType": 1, "query": "label_values(exordos_compute_node_info{element=~\"$element\"}, hostname)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "multi": false, "includeAll": false, "current": {}, "sort": 1 }, { "name": "search", "label": "Search", "type": "textbox", "description": "Extra LogsQL filter for the Node logs panel, e.g. error or _SYSTEMD_UNIT:ec-gservice.service", "query": "", "current": { "text": "", "value": "" }, "hide": 0 } ] }, "panels": [ { "id": 1, "type": "row", "title": "Overview", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [] }, { "id": 2, "type": "stat", "title": "Nodes", "description": "Number of nodes (filtered by the element).", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 4, "w": 3, "x": 0, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "count(exordos_compute_node_status{element=~\"$element\"})", "instant": true, "range": false } ], "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "value", "justifyMode": "center", "orientation": "auto", "showPercentChange": false }, "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "text" }, "unit": "short", "decimals": 0, "noValue": "0" }, "overrides": [] } }, { "id": 3, "type": "stat", "title": "Nodes not ACTIVE", "description": "Nodes in any status but ACTIVE.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 4, "w": 3, "x": 3, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "count(exordos_compute_node_status{element=~\"$element\"} != 5)", "instant": true, "range": false } ], "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "value", "justifyMode": "center", "orientation": "auto", "showPercentChange": false }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short", "decimals": 0, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } }, "overrides": [] } }, { "id": 4, "type": "stat", "title": "Nodes ERROR", "description": "Nodes in ERROR.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 4, "w": 3, "x": 6, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "count(exordos_compute_node_status{element=~\"$element\"} == 6)", "instant": true, "range": false } ], "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "value", "justifyMode": "center", "orientation": "auto", "showPercentChange": false }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short", "decimals": 0, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] } }, { "id": 5, "type": "stat", "title": "Sets", "description": "Number of node sets (filtered by the element).", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 4, "w": 3, "x": 9, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "count(exordos_compute_set_status{element=~\"$element\"})", "instant": true, "range": false } ], "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "value", "justifyMode": "center", "orientation": "auto", "showPercentChange": false }, "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "text" }, "unit": "short", "decimals": 0, "noValue": "0" }, "overrides": [] } }, { "id": 6, "type": "stat", "title": "Degraded sets", "description": "Sets with fewer ACTIVE nodes than replicas.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 4, "w": 3, "x": 12, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "count(((exordos_compute_set_replicas and on(set_uuid) exordos_compute_set_status{element=~\"$element\"}) - on(set_uuid) (count by (set_uuid) (exordos_compute_node_status == 5) or on(set_uuid) exordos_compute_set_replicas * 0)) > 0)", "instant": true, "range": false } ], "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "value", "justifyMode": "center", "orientation": "auto", "showPercentChange": false }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short", "decimals": 0, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] } }, { "id": 7, "type": "stat", "title": "Cores allocated", "description": "Share of the cores of all machine pools taken by machines.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 4, "w": 3, "x": 15, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "1 - sum(exordos_compute_pool_cores_available) / sum(exordos_compute_pool_cores)", "instant": true, "range": false } ], "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "value", "justifyMode": "center", "orientation": "auto", "showPercentChange": false }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "percentunit", "decimals": 1, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 0.9 } ] } }, "overrides": [] } }, { "id": 8, "type": "stat", "title": "RAM allocated", "description": "Share of the RAM of all machine pools taken by machines.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 4, "w": 3, "x": 18, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "1 - sum(exordos_compute_pool_ram_available_bytes) / sum(exordos_compute_pool_ram_bytes)", "instant": true, "range": false } ], "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "value", "justifyMode": "center", "orientation": "auto", "showPercentChange": false }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "percentunit", "decimals": 1, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 0.9 } ] } }, "overrides": [] } }, { "id": 9, "type": "stat", "title": "Data age", "description": "Time since exordos_core last wrote the state metrics. The file is rewritten every 15s; if it grows, ec-gservice is down or stuck and the dashboard shows stale data.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 4, "w": 3, "x": 21, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "time() - max(node_textfile_mtime_seconds{file=~\".*core_state.prom\"})", "instant": true, "range": false } ], "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "value", "justifyMode": "center", "orientation": "auto", "showPercentChange": false }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "s", "decimals": 0, "noValue": "No data", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 60 } ] } }, "overrides": [] } }, { "id": 10, "type": "row", "title": "Nodes", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "panels": [] }, { "id": 11, "type": "table", "title": "Nodes", "description": "Every node. Click a hostname to open its host metrics in Node Exporter Full or its logs; click an element to open it in Exordos Elements. \"Not ACTIVE in range\" is how long the node spent outside ACTIVE within the selected time range.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 12, "w": 24, "x": 0, "y": 6 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max without (instance, job) (exordos_compute_node_info{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (node_uuid) (exordos_compute_node_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (node_uuid) (exordos_compute_node_cores and on(node_uuid) exordos_compute_node_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "D", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (node_uuid) (exordos_compute_node_ram_bytes and on(node_uuid) exordos_compute_node_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "E", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (node_uuid) (exordos_compute_node_disk_bytes and on(node_uuid) exordos_compute_node_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "F", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (node_uuid) (time() - exordos_compute_node_created_timestamp_seconds and on(node_uuid) exordos_compute_node_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "G", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (node_uuid) ((1 - share_eq_over_time(exordos_compute_node_status{element=~\"$element\"}[$__range], 5)) * lifetime(exordos_compute_node_status{element=~\"$element\"}[$__range]) and on(node_uuid) exordos_compute_node_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "H", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (node_uuid) (exordos_compute_node_status{element=~\"$element\"} * (exordos_compute_node_status{element=~\"$element\"} != bool 5))", "format": "table", "instant": true, "range": false } ], "transformations": [ { "id": "merge", "options": {} }, { "id": "sortBy", "options": { "sort": [ { "field": "Value #H", "desc": true } ] } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "Value #A": true, "node_uuid": true, "set_uuid": true, "project_id": true, "Value #H": true }, "indexByName": { "node": 0, "hostname": 1, "Value #B": 2, "description": 3, "set": 4, "element": 5, "node_type": 6, "Value #C": 7, "Value #D": 8, "disks": 9, "Value #E": 10, "image": 11, "ipv4": 12, "pool": 13, "Value #F": 14, "Value #G": 15 }, "renameByName": { "node": "Node", "hostname": "Hostname", "Value #B": "Status", "description": "Description", "set": "Set", "element": "Element", "node_type": "Type", "Value #C": "Cores", "Value #D": "RAM", "disks": "Disks", "Value #E": "Disk total", "image": "Image", "ipv4": "IPv4", "pool": "Pool", "Value #F": "Age", "Value #G": "Not ACTIVE in range" } } } ], "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false } }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "cellOptions": { "type": "auto" } } }, "overrides": [ { "matcher": { "id": "byName", "options": "Status" }, "properties": [ { "id": "mappings", "value": [ { "type": "value", "options": { "1": { "text": "NEW", "color": "blue", "index": 0 }, "2": { "text": "SCHEDULED", "color": "purple", "index": 1 }, "3": { "text": "IN_PROGRESS", "color": "orange", "index": 2 }, "4": { "text": "STARTED", "color": "yellow", "index": 3 }, "5": { "text": "ACTIVE", "color": "green", "index": 4 }, "6": { "text": "ERROR", "color": "red", "index": 5 } } } ] }, { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }, { "id": "custom.width", "value": 120 } ] }, { "matcher": { "id": "byName", "options": "Hostname" }, "properties": [ { "id": "links", "value": [ { "title": "Host metrics of ${__value.raw}", "url": "/d/rYdddlPWk?var-job=node_exporter&var-node=${__value.raw}&${__url_time_range}&var-DS_PROMETHEUS=${DS_Metrics}" }, { "title": "Logs of ${__value.raw}", "url": "/d/exordos-compute/exordos-compute?var-host=${__value.raw}&${__url_time_range}&${DS_Metrics:queryparam}&${DS_Logs:queryparam}&viewPanel=panel-40" } ] } ] }, { "matcher": { "id": "byName", "options": "Element" }, "properties": [ { "id": "links", "value": [ { "title": "Open ${__value.raw} in Exordos Elements", "url": "/d/exordos-elements/exordos-elements?var-element=${__value.raw}&${__url_time_range}&${DS_Metrics:queryparam}" } ] } ] }, { "matcher": { "id": "byName", "options": "RAM" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Disk total" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Age" }, "properties": [ { "id": "unit", "value": "dtdurations" } ] }, { "matcher": { "id": "byName", "options": "Not ACTIVE in range" }, "properties": [ { "id": "unit", "value": "s" }, { "id": "decimals", "value": 0 } ] } ] } }, { "id": 13, "type": "row", "title": "Sets", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, "panels": [] }, { "id": 14, "type": "table", "title": "Sets", "description": "Every node set. \"Missing nodes\" is how many replicas have no ACTIVE node.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 19 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max without (instance, job) (exordos_compute_set_info{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (set_uuid) (exordos_compute_set_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (set_uuid) (exordos_compute_set_replicas and on(set_uuid) exordos_compute_set_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "D", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (set_uuid) ((exordos_compute_set_replicas and on(set_uuid) exordos_compute_set_status{element=~\"$element\"}) - on(set_uuid) (count by (set_uuid) (exordos_compute_node_status == 5) or on(set_uuid) exordos_compute_set_replicas * 0))", "format": "table", "instant": true, "range": false }, { "refId": "E", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (set_uuid) (exordos_compute_set_cores and on(set_uuid) exordos_compute_set_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "F", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (set_uuid) (exordos_compute_set_ram_bytes and on(set_uuid) exordos_compute_set_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "G", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (set_uuid) (time() - exordos_compute_set_created_timestamp_seconds and on(set_uuid) exordos_compute_set_status{element=~\"$element\"})", "format": "table", "instant": true, "range": false }, { "refId": "H", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max by (set_uuid) (exordos_compute_set_status{element=~\"$element\"} * (exordos_compute_set_status{element=~\"$element\"} != bool 5))", "format": "table", "instant": true, "range": false } ], "transformations": [ { "id": "merge", "options": {} }, { "id": "sortBy", "options": { "sort": [ { "field": "Value #H", "desc": true } ] } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "Value #A": true, "set_uuid": true, "project_id": true, "Value #H": true }, "indexByName": { "set": 0, "Value #B": 1, "Value #C": 2, "Value #D": 3, "description": 4, "element": 5, "node_type": 6, "Value #E": 7, "Value #F": 8, "disks": 9, "image": 10, "Value #G": 11 }, "renameByName": { "set": "Set", "Value #B": "Status", "Value #C": "Replicas", "Value #D": "Missing nodes", "description": "Description", "element": "Element", "node_type": "Type", "Value #E": "Cores per node", "Value #F": "RAM per node", "disks": "Disks per node", "image": "Image", "Value #G": "Age" } } } ], "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false } }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "cellOptions": { "type": "auto" } } }, "overrides": [ { "matcher": { "id": "byName", "options": "Status" }, "properties": [ { "id": "mappings", "value": [ { "type": "value", "options": { "1": { "text": "NEW", "color": "blue", "index": 0 }, "2": { "text": "SCHEDULED", "color": "purple", "index": 1 }, "3": { "text": "IN_PROGRESS", "color": "orange", "index": 2 }, "4": { "text": "STARTED", "color": "yellow", "index": 3 }, "5": { "text": "ACTIVE", "color": "green", "index": 4 }, "6": { "text": "ERROR", "color": "red", "index": 5 } } } ] }, { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }, { "id": "custom.width", "value": 120 } ] }, { "matcher": { "id": "byName", "options": "Missing nodes" }, "properties": [ { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, { "id": "custom.cellOptions", "value": { "type": "color-text" } } ] }, { "matcher": { "id": "byName", "options": "Element" }, "properties": [ { "id": "links", "value": [ { "title": "Open ${__value.raw} in Exordos Elements", "url": "/d/exordos-elements/exordos-elements?var-element=${__value.raw}&${__url_time_range}&${DS_Metrics:queryparam}" } ] } ] }, { "matcher": { "id": "byName", "options": "RAM per node" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Age" }, "properties": [ { "id": "unit", "value": "dtdurations" } ] } ] } }, { "id": 16, "type": "row", "title": "Created and deleted", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, "panels": [] }, { "id": 17, "type": "timeseries", "title": "Nodes", "description": "How many nodes appeared and disappeared per interval. Derived from the state metrics, so an object that lived less than the 15s write period can be missed.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, "interval": "1m", "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "count(group by (node_uuid) (present_over_time(exordos_compute_node_status{element=~\"$element\"}[$__interval])) unless on(node_uuid) group by (node_uuid) (exordos_compute_node_status{element=~\"$element\"} offset $__interval)) and on() (count(exordos_compute_node_status{element=~\"$element\"} offset $__interval) > 0)", "range": true, "legendFormat": "Created" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "count(group by (node_uuid) (present_over_time(exordos_compute_node_status{element=~\"$element\"}[$__interval])) unless on(node_uuid) group by (node_uuid) (exordos_compute_node_status{element=~\"$element\"})) and on() (count(exordos_compute_node_status{element=~\"$element\"} offset $__interval) > 0)", "range": true, "legendFormat": "Deleted" } ], "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "noValue": "0", "custom": { "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, "stacking": { "mode": "none", "group": "A" } } }, "overrides": [ { "matcher": { "id": "byName", "options": "Created" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "Deleted" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] } }, { "id": 18, "type": "timeseries", "title": "Sets", "description": "How many sets appeared and disappeared per interval. Derived from the state metrics, so an object that lived less than the 15s write period can be missed.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, "interval": "1m", "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "count(group by (set_uuid) (present_over_time(exordos_compute_set_status{element=~\"$element\"}[$__interval])) unless on(set_uuid) group by (set_uuid) (exordos_compute_set_status{element=~\"$element\"} offset $__interval)) and on() (count(exordos_compute_set_status{element=~\"$element\"} offset $__interval) > 0)", "range": true, "legendFormat": "Created" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "count(group by (set_uuid) (present_over_time(exordos_compute_set_status{element=~\"$element\"}[$__interval])) unless on(set_uuid) group by (set_uuid) (exordos_compute_set_status{element=~\"$element\"})) and on() (count(exordos_compute_set_status{element=~\"$element\"} offset $__interval) > 0)", "range": true, "legendFormat": "Deleted" } ], "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "noValue": "0", "custom": { "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, "stacking": { "mode": "none", "group": "A" } } }, "overrides": [ { "matcher": { "id": "byName", "options": "Created" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byName", "options": "Deleted" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] } }, { "id": 19, "type": "row", "title": "Capacity", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, "panels": [] }, { "id": 20, "type": "table", "title": "Machine pools", "description": "Capacity of every machine pool (hypervisor) as the scheduler sees it. The Pool column of the Nodes table shows where each node runs.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 37 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max without (instance, job) (exordos_compute_pool_status)", "format": "table", "instant": true, "range": false }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max without (instance, job) (exordos_compute_pool_cores)", "format": "table", "instant": true, "range": false }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max without (instance, job) (exordos_compute_pool_cores_available)", "format": "table", "instant": true, "range": false }, { "refId": "D", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max without (instance, job) (1 - exordos_compute_pool_cores_available / exordos_compute_pool_cores)", "format": "table", "instant": true, "range": false }, { "refId": "E", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max without (instance, job) (exordos_compute_pool_ram_bytes)", "format": "table", "instant": true, "range": false }, { "refId": "F", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max without (instance, job) (exordos_compute_pool_ram_available_bytes)", "format": "table", "instant": true, "range": false }, { "refId": "G", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "max without (instance, job) (1 - exordos_compute_pool_ram_available_bytes / exordos_compute_pool_ram_bytes)", "format": "table", "instant": true, "range": false } ], "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "pool_uuid": true }, "indexByName": { "pool": 0, "Value #A": 1, "Value #B": 2, "Value #C": 3, "Value #D": 4, "Value #E": 5, "Value #F": 6, "Value #G": 7 }, "renameByName": { "pool": "Pool", "Value #A": "Status", "Value #B": "Cores", "Value #C": "Cores available", "Value #D": "Cores allocated", "Value #E": "RAM", "Value #F": "RAM available", "Value #G": "RAM allocated" } } }, { "id": "sortBy", "options": { "sort": [ { "field": "Pool" } ] } } ], "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false } }, "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "cellOptions": { "type": "auto" } } }, "overrides": [ { "matcher": { "id": "byName", "options": "Status" }, "properties": [ { "id": "mappings", "value": [ { "type": "value", "options": { "1": { "text": "ACTIVE", "color": "green", "index": 0 }, "2": { "text": "IN_PROGRESS", "color": "orange", "index": 1 }, "3": { "text": "MAINTENANCE", "color": "yellow", "index": 2 }, "4": { "text": "DISABLED", "color": "text", "index": 3 } } } ] }, { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }, { "id": "custom.width", "value": 120 } ] }, { "matcher": { "id": "byName", "options": "RAM" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "RAM available" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Cores allocated" }, "properties": [ { "id": "unit", "value": "percentunit" }, { "id": "min", "value": 0 }, { "id": "max", "value": 1 }, { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 0.9 } ] } }, { "id": "custom.cellOptions", "value": { "type": "gauge", "mode": "basic", "valueDisplayMode": "text" } } ] }, { "matcher": { "id": "byName", "options": "RAM allocated" }, "properties": [ { "id": "unit", "value": "percentunit" }, { "id": "min", "value": 0 }, { "id": "max", "value": 1 }, { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 0.9 } ] } }, { "id": "custom.cellOptions", "value": { "type": "gauge", "mode": "basic", "valueDisplayMode": "text" } } ] } ] } }, { "id": 21, "type": "timeseries", "title": "Allocation over time", "description": "Share of the cores and RAM of each pool taken by machines.", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 44 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "1 - exordos_compute_pool_cores_available / exordos_compute_pool_cores", "range": true, "legendFormat": "{{pool}} cores" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_Metrics}" }, "expr": "1 - exordos_compute_pool_ram_available_bytes / exordos_compute_pool_ram_bytes", "range": true, "legendFormat": "{{pool}} RAM" } ], "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1 } }, "overrides": [] } }, { "id": 22, "type": "row", "title": "Node logs", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 52 }, "panels": [] }, { "id": 40, "type": "logs", "title": "Logs of $host", "description": "journald of the node selected in the Host variable (or opened from the Nodes table), narrowed by the Search variable (LogsQL).", "datasource": { "type": "victoriametrics-logs-datasource", "uid": "${DS_Logs}" }, "gridPos": { "h": 14, "w": 24, "x": 0, "y": 53 }, "targets": [ { "refId": "A", "datasource": { "type": "victoriametrics-logs-datasource", "uid": "${DS_Logs}" }, "expr": "_HOSTNAME:=\"$host\" $search", "queryType": "instant" } ], "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true, "dedupStrategy": "none", "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false } } ] }