From 94283c6165267429deed5141d34eab2066f90d38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Bastos?= Date: Sat, 18 Jul 2026 12:16:36 -0300 Subject: [PATCH 1/4] feat: prometheus metrics endpoint on SN and DN nodes --- admin/kubernetes/k8s_deployment_aws.yml | 8 + admin/kubernetes/k8s_deployment_azure.yml | 8 + admin/kubernetes/k8s_deployment_posix.yml | 8 + docs/prometheus_metrics.md | 115 ++++++++++++ hsds/basenode.py | 4 +- hsds/metrics.py | 211 ++++++++++++++++++++++ hsds/util/azureBlobClient.py | 2 + hsds/util/fileClient.py | 4 + hsds/util/s3Client.py | 2 + pyproject.toml | 1 + requirements.txt | 1 + testall.py | 2 +- tests/unit/metrics_test.py | 143 +++++++++++++++ 13 files changed, 507 insertions(+), 2 deletions(-) create mode 100644 docs/prometheus_metrics.md create mode 100644 hsds/metrics.py create mode 100644 tests/unit/metrics_test.py diff --git a/admin/kubernetes/k8s_deployment_aws.yml b/admin/kubernetes/k8s_deployment_aws.yml index c6ae7da4..8912af85 100644 --- a/admin/kubernetes/k8s_deployment_aws.yml +++ b/admin/kubernetes/k8s_deployment_aws.yml @@ -13,6 +13,12 @@ spec: metadata: labels: app: hsds + annotations: + # scrapes the sn container; add a PodMonitor targeting the sn and dn + # ports to also collect dn metrics (see docs/prometheus_metrics.md) + prometheus.io/scrape: "true" + prometheus.io/port: "5101" + prometheus.io/path: /metrics spec: containers: - name: sn @@ -35,6 +41,7 @@ spec: subPath: override.yml ports: - containerPort: 5101 + name: sn env: - name: NODE_TYPE value: sn @@ -79,6 +86,7 @@ spec: subPath: override.yml ports: - containerPort: 6101 + name: dn env: - name: NODE_TYPE value: dn diff --git a/admin/kubernetes/k8s_deployment_azure.yml b/admin/kubernetes/k8s_deployment_azure.yml index 6ad54c49..be417acd 100644 --- a/admin/kubernetes/k8s_deployment_azure.yml +++ b/admin/kubernetes/k8s_deployment_azure.yml @@ -13,6 +13,12 @@ spec: metadata: labels: app: hsds + annotations: + # scrapes the sn container; add a PodMonitor targeting the sn and dn + # ports to also collect dn metrics (see docs/prometheus_metrics.md) + prometheus.io/scrape: "true" + prometheus.io/port: "5101" + prometheus.io/path: /metrics spec: containers: - name: sn @@ -36,6 +42,7 @@ spec: ports: - containerPort: 5101 + name: sn env: - name: NODE_TYPE value: sn @@ -78,6 +85,7 @@ spec: ports: - containerPort: 6101 + name: dn env: - name: NODE_TYPE value: dn diff --git a/admin/kubernetes/k8s_deployment_posix.yml b/admin/kubernetes/k8s_deployment_posix.yml index ea56117e..b4b20ffe 100644 --- a/admin/kubernetes/k8s_deployment_posix.yml +++ b/admin/kubernetes/k8s_deployment_posix.yml @@ -13,6 +13,12 @@ spec: metadata: labels: app: hsds + annotations: + # scrapes the sn container; add a PodMonitor targeting the sn and dn + # ports to also collect dn metrics (see docs/prometheus_metrics.md) + prometheus.io/scrape: "true" + prometheus.io/port: "5101" + prometheus.io/path: /metrics spec: containers: - name: sn @@ -32,6 +38,7 @@ spec: subPath: override.yml ports: - containerPort: 5101 + name: sn env: - name: NODE_TYPE value: sn @@ -61,6 +68,7 @@ spec: subPath: override.yml ports: - containerPort: 6101 + name: dn env: - name: NODE_TYPE value: dn diff --git a/docs/prometheus_metrics.md b/docs/prometheus_metrics.md new file mode 100644 index 00000000..40916a4e --- /dev/null +++ b/docs/prometheus_metrics.md @@ -0,0 +1,115 @@ +# Prometheus Metrics + +Every HSDS node (SN and DN) serves Prometheus metrics at `GET /metrics` in the +standard text exposition format, implemented with the official +[prometheus_client](https://github.com/prometheus/client_python) library. +The endpoint is always served, even when the node is not in the `READY` state, +so an unhealthy node can still be observed. + +## Why these metrics + +The set maps the four golden signals (traffic, errors, latency, saturation) +onto what actually degrades an HSDS deployment: + +- **Traffic / errors / latency** — `hsds_http_requests_total` and + `hsds_http_request_duration_seconds` are recorded by an aiohttp middleware + on every request, including 4xx/5xx responses raised as exceptions. SN + nodes return 503 when overloaded or when DNs are unreachable, so a rising + `status="503"` rate is the primary "service is unhealthy" signal. +- **Cluster membership** — `hsds_node_ready` and `hsds_active_dn_count` + detect nodes stuck outside `READY` and SNs that lost sight of DNs + (`hsds_active_dn_count` dropping below the expected DN count means reduced + capacity even while requests still succeed). +- **Saturation** — `hsds_tasks_active` vs `hsds_tasks_max`: when active + asyncio tasks exceed `max_task_count`, SNs shed load with 503s. Watching + the ratio tells you *before* it happens. Cache gauges + (`hsds_cache_mem_used_bytes` vs `hsds_cache_mem_target_bytes`) expose + chunk/meta/domain cache pressure on DNs. +- **Durability risk** — `hsds_cache_dirty_items` counts objects modified in + memory but not yet flushed to storage. A persistently growing value means + writes are outpacing storage flushes; those bytes are lost if the pod dies. +- **Storage backend** — `hsds_storage_errors_total`, + `hsds_storage_bytes_read_total`, `hsds_storage_bytes_written_total` + (labelled with `backend="s3"|"azure"|"file"`) surface failures and + throughput of the object store, HSDS's hardest dependency. +- **Log-level counters** — `hsds_log_events_total{level="WARN"|"ERROR"}` is a + cheap catch-all: alerting on ERROR rate catches problems that have no + dedicated metric yet. + +The default registry also provides `process_*` and `python_gc_*` metrics +(CPU, RSS, FDs, GC) at no extra cost. + +## Metric reference + +| Metric | Type | Labels | Source | +|---|---|---|---| +| `hsds_info` | gauge | `node_type`, `node_id`, `version` | node identity, useful for version rollout tracking | +| `hsds_node_ready` | gauge | | 1 when node state is `READY` | +| `hsds_start_time_seconds` | gauge | | node start time (restart detection) | +| `hsds_active_dn_count` | gauge | | DN urls this node currently knows about | +| `hsds_tasks_active` | gauge | | current asyncio task count | +| `hsds_tasks_max` | gauge | | task count above which SNs return 503 | +| `hsds_http_requests_total` | counter | `method`, `status` | all HTTP requests, via middleware | +| `hsds_http_request_duration_seconds` | histogram | | request latency, via middleware | +| `hsds_log_events_total` | counter | `level` | WARN/ERROR log line counts | +| `hsds_cache_items` | gauge | `cache` (`meta`/`chunk`/`domain`) | objects held in cache | +| `hsds_cache_dirty_items` | gauge | `cache` | objects awaiting flush to storage | +| `hsds_cache_mem_used_bytes` | gauge | `cache` | cache memory used | +| `hsds_cache_mem_target_bytes` | gauge | `cache` | configured cache memory target | +| `hsds_storage_errors_total` | counter | `backend` | storage client errors | +| `hsds_storage_bytes_read_total` | counter | `backend` | bytes read from storage | +| `hsds_storage_bytes_written_total` | counter | `backend` | bytes written to storage | + +## Scraping on Kubernetes + +The deployment manifests in `admin/kubernetes/` set the conventional pod +annotations (`prometheus.io/scrape: "true"`, `prometheus.io/port: "5101"`), +which scrape the SN container. Annotation-based discovery only supports one +port per pod; to also scrape the DN container use a PodMonitor +(Prometheus Operator) targeting both named container ports: + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: hsds +spec: + selector: + matchLabels: + app: hsds + podMetricsEndpoints: + - port: sn + path: /metrics + - port: dn + path: /metrics +``` + +## Suggested alerts + +```yaml +groups: + - name: hsds + rules: + - alert: HsdsNodeNotReady + expr: hsds_node_ready == 0 + for: 5m + - alert: HsdsHighErrorRate + expr: > + sum(rate(hsds_http_requests_total{status=~"5.."}[5m])) + / sum(rate(hsds_http_requests_total[5m])) > 0.05 + for: 10m + - alert: HsdsSlowRequests + expr: > + histogram_quantile(0.99, + rate(hsds_http_request_duration_seconds_bucket[5m])) > 5 + for: 10m + - alert: HsdsTaskSaturation + expr: hsds_tasks_active / hsds_tasks_max > 0.8 + for: 5m + - alert: HsdsDirtyCacheGrowing + expr: sum by (pod) (hsds_cache_dirty_items) > 100 + for: 15m + - alert: HsdsStorageErrors + expr: rate(hsds_storage_errors_total[5m]) > 0 + for: 5m +``` diff --git a/hsds/basenode.py b/hsds/basenode.py index 69a00531..7caf01a1 100644 --- a/hsds/basenode.py +++ b/hsds/basenode.py @@ -32,6 +32,7 @@ from .util.authUtil import isAdminUser from .util.k8sClient import getDnLabelSelector, getPodIps from . import hsds_logger as log +from . import metrics HSDS_VERSION = "0.9.4" @@ -561,7 +562,7 @@ def baseInit(node_type): # create the app object log.info("Application baseInit") - app = Application() + app = Application(middlewares=[metrics.metrics_middleware]) app["node_state"] = "INITIALIZING" app["node_number"] = -1 @@ -701,6 +702,7 @@ def baseInit(node_type): app.router.add_get("/info", info) app.router.add_get("/about", about) + app.router.add_get("/metrics", metrics.metrics_handler) if is_standalone: # can go straight to ready state diff --git a/hsds/metrics.py b/hsds/metrics.py new file mode 100644 index 00000000..b05c584e --- /dev/null +++ b/hsds/metrics.py @@ -0,0 +1,211 @@ +############################################################################## +# Copyright by The HDF Group. # +# All rights reserved. # +# # +# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # +# Utilities. The full HSDS copyright notice, including # +# terms governing use, modification, and redistribution, is contained in # +# the file COPYING, which can be found at the root of the source code # +# distribution tree. If you do not have access to this file, you may # +# request a copy from help@hdfgroup.org. # +############################################################################## +# +# Prometheus metrics exporter - serves /metrics in text exposition format +# +import asyncio + +from aiohttp import web +from prometheus_client import ( + CONTENT_TYPE_LATEST, + REGISTRY, + CollectorRegistry, + Counter, + Histogram, + generate_latest, +) +from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily + +from . import hsds_logger as log + +# request duration histogram bucket upper bounds, in seconds +BUCKETS = (0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0) + +_CACHES = ("meta_cache", "chunk_cache", "domain_cache") +_STORAGE_STATS = ("s3_stats", "azure_stats", "file_stats") + + +def _make_http_metrics(): + requests = Counter( + "hsds_http_requests", + "HTTP requests by method and status", + ["method", "status"], + ) + duration = Histogram( + "hsds_http_request_duration_seconds", + "HTTP request latency", + buckets=BUCKETS, + ) + return requests, duration + + +HTTP_REQUESTS, HTTP_DURATION = _make_http_metrics() + + +def reset(): + """re-create the http metrics with zeroed counts""" + global HTTP_REQUESTS, HTTP_DURATION + REGISTRY.unregister(HTTP_REQUESTS) + REGISTRY.unregister(HTTP_DURATION) + HTTP_REQUESTS, HTTP_DURATION = _make_http_metrics() + + +def observe_request(method, status, duration): + """record one completed http request""" + HTTP_REQUESTS.labels(method=method, status=str(int(status))).inc() + HTTP_DURATION.observe(duration) + + +@web.middleware +async def metrics_middleware(request, handler): + """count every request and its latency, including error responses""" + loop = asyncio.get_event_loop() + start = loop.time() + status = 500 + try: + resp = await handler(request) + status = resp.status + return resp + except web.HTTPException as he: + status = he.status + raise + finally: + observe_request(request.method, status, loop.time() - start) + + +class AppStateCollector: + """expose state the app already tracks (node, caches, storage counters) + as Prometheus metrics, read at scrape time""" + + def __init__(self, app): + self._app = app + + def collect(self): + from .basenode import getVersion # deferred to avoid circular import + + app = self._app + + info = GaugeMetricFamily( + "hsds_info", + "Node identity: type, id and hsds version", + labels=["node_type", "node_id", "version"], + ) + node_type = app.get("node_type", "") + info.add_metric([node_type, app.get("id", ""), getVersion()], 1) + yield info + + ready = 1 if app.get("node_state") == "READY" else 0 + yield GaugeMetricFamily( + "hsds_node_ready", "1 if node state is READY, 0 otherwise", value=ready + ) + + if "start_time" in app: + yield GaugeMetricFamily( + "hsds_start_time_seconds", + "Unix time the node started", + value=app["start_time"], + ) + + if "dn_urls" in app: + yield GaugeMetricFamily( + "hsds_active_dn_count", + "Data nodes this node currently knows about", + value=len(app["dn_urls"]), + ) + + try: + num_tasks = len(asyncio.all_tasks()) + except RuntimeError: + num_tasks = 0 # no running event loop + yield GaugeMetricFamily( + "hsds_tasks_active", "Current asyncio task count", value=num_tasks + ) + + if app.get("max_task_count"): + yield GaugeMetricFamily( + "hsds_tasks_max", + "Task count above which SN nodes return 503", + value=app["max_task_count"], + ) + + log_events = CounterMetricFamily( + "hsds_log_events", "Log events by level", labels=["level"] + ) + for level in ("WARN", "ERROR"): + log_events.add_metric([level], log.log_count.get(level, 0)) + yield log_events + + caches = [(key[: -len("_cache")], app[key]) for key in _CACHES if key in app] + if caches: + items = GaugeMetricFamily( + "hsds_cache_items", "Objects held in cache", labels=["cache"] + ) + dirty = GaugeMetricFamily( + "hsds_cache_dirty_items", + "Cache objects not yet flushed to storage", + labels=["cache"], + ) + mem_used = GaugeMetricFamily( + "hsds_cache_mem_used_bytes", "Cache memory used", labels=["cache"] + ) + mem_target = GaugeMetricFamily( + "hsds_cache_mem_target_bytes", "Cache memory target", labels=["cache"] + ) + for name, cache in caches: + items.add_metric([name], len(cache)) + dirty.add_metric([name], cache.dirtyCount) + mem_used.add_metric([name], cache.memUsed) + mem_target.add_metric([name], cache.memTarget) + yield items + yield dirty + yield mem_used + yield mem_target + + for key in _STORAGE_STATS: + if key not in app: + continue + backend = key[: -len("_stats")] + stats = app[key] + errors = CounterMetricFamily( + "hsds_storage_errors", "Storage operation errors", labels=["backend"] + ) + errors.add_metric([backend], stats.get("error_count", 0)) + yield errors + bytes_read = CounterMetricFamily( + "hsds_storage_bytes_read", "Bytes read from storage", labels=["backend"] + ) + bytes_read.add_metric([backend], stats.get("bytes_in", 0)) + yield bytes_read + bytes_written = CounterMetricFamily( + "hsds_storage_bytes_written", + "Bytes written to storage", + labels=["backend"], + ) + bytes_written.add_metric([backend], stats.get("bytes_out", 0)) + yield bytes_written + break # only one storage backend is active per node + + +def render(app): + """render app state plus process/http metrics as Prometheus text format""" + app_registry = CollectorRegistry() + app_registry.register(AppStateCollector(app)) + text = generate_latest(app_registry).decode("utf-8") + text += generate_latest(REGISTRY).decode("utf-8") + return text + + +async def metrics_handler(request): + """HTTP GET handler for /metrics - Prometheus scrape endpoint""" + text = render(request.app) + headers = {"Content-Type": CONTENT_TYPE_LATEST} + return web.Response(body=text.encode("utf-8"), headers=headers) diff --git a/hsds/util/azureBlobClient.py b/hsds/util/azureBlobClient.py index ba9a2eab..b698e570 100644 --- a/hsds/util/azureBlobClient.py +++ b/hsds/util/azureBlobClient.py @@ -152,6 +152,8 @@ async def get_object(self, key, bucket=None, offset=0, length=-1): log.error(msg) raise HTTPInternalServerError() + if data and len(data) > 0: + self._azure_stats_increment("bytes_in", inc=len(data)) return data async def put_object(self, key, data, bucket=None): diff --git a/hsds/util/fileClient.py b/hsds/util/fileClient.py index 4e9c5ef0..4ebf704e 100644 --- a/hsds/util/fileClient.py +++ b/hsds/util/fileClient.py @@ -108,6 +108,8 @@ def _file_stats_increment(self, counter, inc=1): log.error(f"unexpected inc for file_stats: {inc}") return + file_stats[counter] += inc + def getURIFromKey(self, key, bucket=None): """ return filesystem specific URI for given key and bucket """ if not bucket: @@ -173,6 +175,8 @@ async def get_object(self, key, bucket=None, offset=0, length=-1): msg = f"Unexpected Exception {type(e)} get get_object {key}: {e}" log.error(msg) raise HTTPInternalServerError() + if data and len(data) > 0: + self._file_stats_increment("bytes_in", inc=len(data)) return data async def put_object(self, key, data, bucket=None): diff --git a/hsds/util/s3Client.py b/hsds/util/s3Client.py index 4c6fe9e2..f6de034f 100644 --- a/hsds/util/s3Client.py +++ b/hsds/util/s3Client.py @@ -350,6 +350,8 @@ async def get_object(self, key, bucket=None, offset=0, length=-1): msg = f"Unexpected Exception {type(e)} get s3 obj {bucket}/{key}: {e}" log.error(msg) raise HTTPInternalServerError() + if data and len(data) > 0: + self._s3_stats_increment("bytes_in", inc=len(data)) return data async def put_object(self, key, data, bucket=None): diff --git a/pyproject.toml b/pyproject.toml index 40e17d74..77f05cbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ "importlib_resources", "numcodecs <= 0.15.1", "numpy >=2.0.0", + "prometheus-client", "psutil", "pyjwt", "pytz", diff --git a/requirements.txt b/requirements.txt index 76420509..3d150b95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ multidict==6.6.4 numcodecs==0.13.1 numpy==2.2.6 packaging==25.0 +prometheus_client==0.25.0 propcache==0.3.2 psutil==7.0.0 pycparser==2.22 diff --git a/testall.py b/testall.py index 1e8ea348..e10b9e52 100755 --- a/testall.py +++ b/testall.py @@ -17,7 +17,7 @@ unit_tests = ('array_util_test', 'chunk_util_test', 'compression_test', 'domain_util_test', 'dset_util_test', 'hdf5_dtype_test', 'id_util_test', 'lru_cache_test', - 'shuffle_test', 'rangeget_util_test') + 'metrics_test', 'shuffle_test', 'rangeget_util_test') integ_tests = ('uptest', 'setup_test', 'domain_test', 'group_test', 'link_test', 'attr_test', 'datatype_test', 'dataset_test', diff --git a/tests/unit/metrics_test.py b/tests/unit/metrics_test.py new file mode 100644 index 00000000..e95db73d --- /dev/null +++ b/tests/unit/metrics_test.py @@ -0,0 +1,143 @@ +############################################################################## +# Copyright by The HDF Group. # +# All rights reserved. # +# # +# This file is part of HSDS (HDF5 Scalable Data Service), Libraries and # +# Utilities. The full HSDS copyright notice, including # +# terms governing use, modification, and redistribution, is contained in # +# the file COPYING, which can be found at the root of the source code # +# distribution tree. If you do not have access to this file, you may # +# request a copy from help@hdfgroup.org. # +############################################################################## +import asyncio +import sys +import unittest + +from aiohttp import web +from aiohttp.test_utils import make_mocked_request + +sys.path.append("../..") +from hsds import metrics +from hsds.util.fileClient import FileClient +from hsds.util.lruCache import LruCache + + +def parse(text): + """parse Prometheus exposition text into a {series: value} dict""" + samples = {} + for line in text.splitlines(): + if not line or line.startswith("#"): + continue + key, value = line.rsplit(" ", 1) + samples[key] = float(value) + return samples + + +class FileClientStatsTest(unittest.TestCase): + """storage metrics depend on the file_stats counters actually counting""" + + def testStatsIncrement(self): + client = FileClient.__new__(FileClient) # skip __init__, needs root_dir config + client._app = {} + client._file_stats_increment("bytes_out", inc=42) + client._file_stats_increment("bytes_out", inc=8) + client._file_stats_increment("error_count") + stats = client._app["file_stats"] + self.assertEqual(stats["bytes_out"], 50) + self.assertEqual(stats["error_count"], 1) + + +class MetricsTest(unittest.TestCase): + def setUp(self): + metrics.reset() + + def testHttpRequestMetrics(self): + metrics.observe_request("GET", 200, 0.05) + metrics.observe_request("GET", 200, 0.2) + metrics.observe_request("PUT", 503, 2.0) + text = metrics.render({}) + samples = parse(text) + self.assertEqual(samples['hsds_http_requests_total{method="GET",status="200"}'], 2) + self.assertEqual(samples['hsds_http_requests_total{method="PUT",status="503"}'], 1) + self.assertEqual(samples["hsds_http_request_duration_seconds_count"], 3) + self.assertAlmostEqual(samples["hsds_http_request_duration_seconds_sum"], 2.25) + # histogram buckets are cumulative + self.assertEqual(samples['hsds_http_request_duration_seconds_bucket{le="0.1"}'], 1) + self.assertEqual(samples['hsds_http_request_duration_seconds_bucket{le="0.25"}'], 2) + self.assertEqual(samples['hsds_http_request_duration_seconds_bucket{le="+Inf"}'], 3) + + def testNodeMetrics(self): + app = { + "node_state": "READY", + "node_type": "sn", + "id": "sn-001", + "start_time": 1234567890, + "dn_urls": ["http://dn1:6101", "http://dn2:6101"], + "max_task_count": 100, + } + text = metrics.render(app) + samples = parse(text) + self.assertEqual(samples["hsds_node_ready"], 1) + self.assertEqual(samples["hsds_start_time_seconds"], 1234567890) + self.assertEqual(samples["hsds_active_dn_count"], 2) + self.assertEqual(samples["hsds_tasks_max"], 100) + self.assertIn('node_type="sn"', text) + self.assertIn("hsds_tasks_active", text) + self.assertIn('hsds_log_events_total{level="ERROR"}', text) + self.assertIn('hsds_log_events_total{level="WARN"}', text) + + app["node_state"] = "TERMINATING" + samples = parse(metrics.render(app)) + self.assertEqual(samples["hsds_node_ready"], 0) + + def testCacheMetrics(self): + app = {"chunk_cache": LruCache(mem_target=1000)} + samples = parse(metrics.render(app)) + self.assertEqual(samples['hsds_cache_items{cache="chunk"}'], 0) + self.assertEqual(samples['hsds_cache_dirty_items{cache="chunk"}'], 0) + self.assertEqual(samples['hsds_cache_mem_used_bytes{cache="chunk"}'], 0) + self.assertEqual(samples['hsds_cache_mem_target_bytes{cache="chunk"}'], 1000) + + def testStorageMetrics(self): + s3_stats = {"error_count": 1, "bytes_in": 100, "bytes_out": 50} + app = {"s3_stats": s3_stats} + samples = parse(metrics.render(app)) + self.assertEqual(samples['hsds_storage_errors_total{backend="s3"}'], 1) + self.assertEqual(samples['hsds_storage_bytes_read_total{backend="s3"}'], 100) + self.assertEqual(samples['hsds_storage_bytes_written_total{backend="s3"}'], 50) + + def testMiddleware(self): + async def run(): + async def ok_handler(request): + return web.Response(status=204) + + async def err_handler(request): + raise web.HTTPServiceUnavailable() + + req = make_mocked_request("GET", "/domains") + resp = await metrics.metrics_middleware(req, ok_handler) + self.assertEqual(resp.status, 204) + + req = make_mocked_request("PUT", "/domains") + with self.assertRaises(web.HTTPServiceUnavailable): + await metrics.metrics_middleware(req, err_handler) + + asyncio.run(run()) + samples = parse(metrics.render({})) + self.assertEqual(samples['hsds_http_requests_total{method="GET",status="204"}'], 1) + self.assertEqual(samples['hsds_http_requests_total{method="PUT",status="503"}'], 1) + + def testMetricsHandler(self): + async def run(): + app = {"node_state": "READY"} + req = make_mocked_request("GET", "/metrics", app=app) + resp = await metrics.metrics_handler(req) + self.assertEqual(resp.status, 200) + self.assertTrue(resp.headers["Content-Type"].startswith("text/plain")) + self.assertIn(b"hsds_node_ready 1", resp.body) + + asyncio.run(run()) + + +if __name__ == "__main__": + unittest.main() From aa0ed3b278d02400bc7a0755730423bb83901f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Bastos?= Date: Mon, 27 Jul 2026 09:37:44 -0300 Subject: [PATCH 2/4] docs: front HSDS with ingress/gateway, restrict internal endpoints, and autoscaling --- admin/kubernetes/k8s_gateway_envoy.yml | 61 +++++++++++++ admin/kubernetes/k8s_ingress_nginx.yml | 36 ++++++++ admin/kubernetes/k8s_service_lb.yml | 24 ----- admin/kubernetes/k8s_service_lb_azure.yml | 14 --- docs/design/kubernetes/hsds_kubernetes.md | 104 +++++++++++++++++++--- docs/kubernetes_install_aws.md | 25 +++--- docs/kubernetes_install_azure.md | 23 ++--- docs/prometheus_metrics.md | 20 +++++ 8 files changed, 235 insertions(+), 72 deletions(-) create mode 100644 admin/kubernetes/k8s_gateway_envoy.yml create mode 100644 admin/kubernetes/k8s_ingress_nginx.yml delete mode 100644 admin/kubernetes/k8s_service_lb.yml delete mode 100644 admin/kubernetes/k8s_service_lb_azure.yml diff --git a/admin/kubernetes/k8s_gateway_envoy.yml b/admin/kubernetes/k8s_gateway_envoy.yml new file mode 100644 index 00000000..1d2b68f4 --- /dev/null +++ b/admin/kubernetes/k8s_gateway_envoy.yml @@ -0,0 +1,61 @@ +# Expose the public HSDS API through Gateway API (Envoy Gateway) while blocking the +# internal endpoints (/metrics, /info, /about) from external callers. +# +# In-cluster access is unaffected (see the note in k8s_ingress_nginx.yml): scrapers hit +# pod IPs directly and in-cluster clients use hsds..svc.cluster.local:5101, +# neither of which traverses this Gateway. +# +# Requires Envoy Gateway v1.3+ for the HTTPRouteFilter `directResponse` extension. +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: hsds +spec: + gatewayClassName: eg # GatewayClass created by Envoy Gateway + listeners: + - name: http + protocol: HTTP + port: 80 + hostname: hsds.example.com +--- +# Envoy Gateway extension filter: return a fixed 403 response with no backend call. +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: HTTPRouteFilter +metadata: + name: deny-403 +spec: + directResponse: + statusCode: 403 + body: + type: Inline + inline: "Forbidden\n" +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: hsds +spec: + parentRefs: + - name: hsds + hostnames: + - hsds.example.com + rules: + # Exact matches take precedence over the PathPrefix catch-all below (Gateway API + # match precedence), so these internal paths get a 403 before reaching the backend. + - matches: + - path: {type: Exact, value: /metrics} + - path: {type: Exact, value: /info} + - path: {type: Exact, value: /about} + filters: + - type: ExtensionRef + extensionRef: + group: gateway.envoyproxy.io + kind: HTTPRouteFilter + name: deny-403 + # Public API -> SN service. + - matches: + - path: {type: PathPrefix, value: /} + backendRefs: + - name: hsds + port: 5101 diff --git a/admin/kubernetes/k8s_ingress_nginx.yml b/admin/kubernetes/k8s_ingress_nginx.yml new file mode 100644 index 00000000..179367bd --- /dev/null +++ b/admin/kubernetes/k8s_ingress_nginx.yml @@ -0,0 +1,36 @@ +# Expose the public HSDS API through ingress-nginx while blocking the internal +# endpoints (/metrics, /info, /about) from external callers. +# +# In-cluster access is unaffected: a Prometheus PodMonitor scrapes pod IPs directly, +# and in-cluster clients reach the SN service at +# hsds..svc.cluster.local:5101 +# Neither path goes through this Ingress, so the deny rule below only affects +# north-south (external) traffic. The DN container (:6101) is never in a Service, so +# it is only ever reachable in-cluster. +# +# NOTE: server-snippet requires `allow-snippet-annotations: "true"` in the +# ingress-nginx controller ConfigMap (disabled by default since v1.9.0, CVE-2023-5044). +# If you can't enable snippets, keep Prometheus in-cluster and simply don't add an +# Ingress for these endpoints — they stay reachable only via the ClusterIP service. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hsds + annotations: + nginx.ingress.kubernetes.io/server-snippet: | + location ~* "^/(metrics|info|about)$" { + deny all; # returns 403 Forbidden + } +spec: + ingressClassName: nginx + rules: + - host: hsds.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: hsds + port: + number: 5101 diff --git a/admin/kubernetes/k8s_service_lb.yml b/admin/kubernetes/k8s_service_lb.yml deleted file mode 100644 index 59eba4b4..00000000 --- a/admin/kubernetes/k8s_service_lb.yml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: hsds - labels: - app: hsds - annotations: - # Note that the backend talks over HTTP. - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http - # Incudes ARN of the certificate. - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-west-2:1234567:certificate/abcdefgh-85ca-42ab-a5f3-8761a8dfe77d - # Only run SSL on the port named "https" below. - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" -spec: - selector: - app: hsds - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 5101 - - name: https - port: 443 - targetPort: 5101 \ No newline at end of file diff --git a/admin/kubernetes/k8s_service_lb_azure.yml b/admin/kubernetes/k8s_service_lb_azure.yml deleted file mode 100644 index 8cce4a9c..00000000 --- a/admin/kubernetes/k8s_service_lb_azure.yml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: hsds - labels: - app: hsds -spec: - selector: - app: hsds - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 5101 \ No newline at end of file diff --git a/docs/design/kubernetes/hsds_kubernetes.md b/docs/design/kubernetes/hsds_kubernetes.md index 39359b7b..d04eef96 100644 --- a/docs/design/kubernetes/hsds_kubernetes.md +++ b/docs/design/kubernetes/hsds_kubernetes.md @@ -60,15 +60,7 @@ The following diagram illustrates the state transitions for ndoes: If external access to the HSDS is not needed (example it will only be using by other K8s applications), load balancing between the different SN nodes will be via the K8S ClusterIP internal load balancer. -If external access is desired, an external load balancer will be used. For AWS, the following annotations to the hsds service yaml can be used: - - annotations: - # Note that the backend talks over HTTP. - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http - # Incudes ARN of the certificate. - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:region:acctid:certificate/load_balancer_id - # Only run SSL on the port named "https" below. - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" +If external access is desired, expose the `hsds` service through an Ingress or the Gateway API. This is also where you terminate TLS and restrict the internal endpoints (`/metrics`, `/info`, `/about`) so they return `403` for outside callers while staying reachable in-cluster. See [Restricting external access](../../prometheus_metrics.md#restricting-external-access-to-metrics-and-info) and the example manifests `admin/kubernetes/k8s_ingress_nginx.yml` (ingress-nginx) and `admin/kubernetes/k8s_gateway_envoy.yml` (Gateway API + Envoy Gateway). ## 6. Secrets @@ -96,6 +88,94 @@ When enabled Cluster Autoscaling (CA) has the effect that more VMs will be launc Cluster Autoscaling by itself will not scale up the number of HSDS pods when either the number of clients is excessively high, or a few number of clients are triggering a significant amount of work (e.g. selection requests that span large number of chunks). To resolve this issue we need to setup Horizontal Autoscaling (HA) that scales up or down the number of HSDS pods based on a specific criteria. -Common metrics used with HA or CPU utilization or memory usaage, however for HSDS are better criteria would be when 503 (Server too busy) http responses are returned to the client. Each HSDS node is configured to handle a specific number of inflight requests (defaulting to 100). When this number is exceeded, a 503 response is returned. Client libraries such as h5pyd know to use this as a signal to scale back the amount of requests being sent to HSDS. - -This will require the development of a custom autoscaler for 503 responses along with the configuration of approriatte heuresitcs for scalee up and scale down. Scale up events that exceed the cluster capacity (leaving pods in pending state), will trigger CA to scale up the number of machines. +Common metrics used with HA are CPU utilization or memory usage, however for HSDS a better criteria is when 503 (Server too busy) http responses are returned to the client. Each HSDS node is configured to handle a specific number of inflight requests (`max_task_count`, defaulting to 100). When this number is exceeded, a 503 response is returned. Client libraries such as h5pyd know to use this as a signal to scale back the amount of requests being sent to HSDS. Task saturation (`hsds_tasks_active / hsds_tasks_max`) is the same signal one step earlier, before any 503 is returned. + +Both signals are now exported as Prometheus metrics (see [prometheus_metrics.md](../../prometheus_metrics.md)): `hsds_http_requests_total{status="503"}` and `hsds_tasks_active` / `hsds_tasks_max`. This means the custom 503 autoscaler this section originally called for is **no longer needed** — a standard HPA or KEDA can consume these metrics directly. Two approaches, simplest first: + +### 9.1 HPA with metrics-server (+ prometheus-adapter) + +**metrics-server** feeds CPU/memory to a plain `HorizontalPodAutoscaler`. This is a coarse starting point (CPU rises under load) that needs no Prometheus: + +```yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: hsds +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: hsds + minReplicas: 2 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 +``` + +**prometheus-adapter** exposes the HSDS metrics through the `custom.metrics.k8s.io` API so the same HPA can scale on the real signal — the per-pod 503 rate: + +```yaml +# prometheus-adapter rule (values.yaml): surface the 503 rate per pod +rules: + - seriesQuery: 'hsds_http_requests_total{status="503"}' + resources: {overrides: {pod: {resource: pod}}} + name: {as: "hsds_http_503_rate"} + metricsQuery: 'sum(rate(hsds_http_requests_total{status="503"}[2m])) by (pod)' +``` + +```yaml +# HPA metric block scaling on the custom metric +metrics: + - type: Pods + pods: + metric: {name: hsds_http_503_rate} + target: + type: AverageValue + averageValue: "1" # scale up when 503s exceed ~1/s/pod +``` + +Good for clusters that already run metrics-server and want HPA without extra operators. Limits: combining more than one metric in a single HPA is awkward, scale-down tuning is coarse, and prometheus-adapter rules are cluster-global config. + +### 9.2 KEDA ScaledObject (more robust) + +[KEDA](https://keda.sh) adds a Prometheus scaler and richer scaling behaviour (multiple triggers, independent up/down cooldowns, replica floors) and manages the underlying HPA for you: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: hsds +spec: + scaleTargetRef: + name: hsds # the hsds Deployment + minReplicaCount: 2 + maxReplicaCount: 20 + cooldownPeriod: 300 # wait 5m of calm before scaling down + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + triggers: + # 1) shed-load signal: cluster-wide 503 rate + - type: prometheus + metadata: + serverAddress: http://prometheus.monitoring.svc.cluster.local:9090 + query: sum(rate(hsds_http_requests_total{status="503"}[2m])) + threshold: "1" + # 2) leading signal: task saturation before 503s appear + - type: prometheus + metadata: + serverAddress: http://prometheus.monitoring.svc.cluster.local:9090 + query: max(hsds_tasks_active / hsds_tasks_max) + threshold: "0.8" +``` + +Good for production: KEDA scales on *either* trigger, its explicit scale-down stabilization avoids flapping on HSDS's bursty selection workloads, and it needs no cluster-global adapter config. + +In both approaches, scale-up events that outrun cluster capacity leave pods in the pending state, which triggers Cluster Autoscaling (section 8) to add machines. diff --git a/docs/kubernetes_install_aws.md b/docs/kubernetes_install_aws.md index ab53ef90..f4cffeda 100644 --- a/docs/kubernetes_install_aws.md +++ b/docs/kubernetes_install_aws.md @@ -111,18 +111,19 @@ If you need to build and deploy a custom HSDS image (e.g. you have made changes Note: if you plan to run HSDS in its own Kubernetes namespace, modify the namespace key of ClusterRoleBinding in k8s_rbac.yml from "default" to your namespace. -2. Create HSDS service: `kubectl apply -f admin/kubernetes/k8s_service_lb.yml` -3. This will create an external load balancer with an http endpoint with a public-ip. - Use kubectl to get the public-ip of the hsds service: `kubectl get service` - You should see an entry similar to: - - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - hsds LoadBalancer 10.0.242.109 20.36.17.252 80:30326/TCP 23 - - Note: the public-ip (EXTERNAL-IP). This is where you can access the HSDS service externally. It may take some time for the EXTERNAL-IP to show up after the service deployment. - - Note: if the service will only be accessed by other pods in the cluster, you can replace - "k8s_service_lb.yml" with "k8s_service.yml" in the kubectl command above. +2. Create the HSDS service: `kubectl apply -f admin/kubernetes/k8s_service.yml` + This is a ClusterIP service exposing the SN port (5101) inside the cluster as + `hsds..svc.cluster.local`. If HSDS will only be accessed by other pods in + the cluster, this is all you need. + +3. For external access, place an Ingress or Gateway in front of the service. This is also + where you terminate TLS and block the internal `/metrics`, `/info`, and `/about` + endpoints from external callers. See the example manifests + `admin/kubernetes/k8s_ingress_nginx.yml` (ingress-nginx) and + `admin/kubernetes/k8s_gateway_envoy.yml` (Gateway API + Envoy Gateway), and + [Restricting external access](prometheus_metrics.md#restricting-external-access-to-metrics-and-info). + The public `EXTERNAL-IP` will belong to your ingress controller / gateway + (`kubectl get service -n `), not the hsds service. 4. Now we will deploy the HSDS pod. In **_k8s_deployment_aws.yml_**, modify the image value if a custom build is being used. E.g: diff --git a/docs/kubernetes_install_azure.md b/docs/kubernetes_install_azure.md index 0d914f8e..f353a494 100644 --- a/docs/kubernetes_install_azure.md +++ b/docs/kubernetes_install_azure.md @@ -82,16 +82,19 @@ delegate operations to other HSDS pods running in the same namespace and same ap ## Create Kubernetes service for HSDS -1. Create HSDS service on the AKS cluster: `$ kubectl apply -f k8s_service_lb_azure.yml` -2. This will create an external load balancer with an http endpoint with a public-ip. - Use kubectl to get the public-ip of the hsds service: `$ kubectl get service` - You should see an entry similar to: - - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - hsds LoadBalancer 10.0.242.109 20.36.17.252 80:30326/TCP 23 - - Note the public-ip (EXTERNAL-IP). This is where you can access the HSDS service externally. It may take some time for the EXTERNAL-IP to show up after the service deployment. For additional configuration options to handle SSL related scenarios please see: _[Front Door Install](frontdoor_install_azure.md)_ - Additional reference for Azure Front Door +1. Create the HSDS service on the AKS cluster: `$ kubectl apply -f k8s_service.yml` + This is a ClusterIP service exposing the SN port (5101) inside the cluster as + `hsds..svc.cluster.local`. If HSDS will only be accessed by other pods in + the cluster, this is all you need. + +2. For external access, place an Ingress or Gateway in front of the service. This is also + where you terminate TLS and block the internal `/metrics`, `/info`, and `/about` + endpoints from external callers. See the example manifests + `admin/kubernetes/k8s_ingress_nginx.yml` (ingress-nginx) and + `admin/kubernetes/k8s_gateway_envoy.yml` (Gateway API + Envoy Gateway), and + [Restricting external access](prometheus_metrics.md#restricting-external-access-to-metrics-and-info). + For Azure-specific SSL / Front Door options see _[Front Door Install](frontdoor_install_azure.md)_ + (). diff --git a/docs/prometheus_metrics.md b/docs/prometheus_metrics.md index 40916a4e..b50e65b5 100644 --- a/docs/prometheus_metrics.md +++ b/docs/prometheus_metrics.md @@ -84,6 +84,26 @@ spec: path: /metrics ``` +## Restricting external access to `/metrics` and `/info` + +`/metrics` and `/info` (and `/about`) expose operational detail — node topology, +versions, cache and storage internals — that should be reachable by in-cluster scrapers +but not from the public internet. + +These endpoints stay fully usable **inside the cluster**: a Prometheus Operator +`PodMonitor` scrapes pod IPs directly, and any in-cluster client can reach the SN +service at `hsds..svc.cluster.local:5101`. Neither path traverses the +Ingress/Gateway, so locking down the public edge doesn't affect them. The DN container +(`:6101`) is never placed in a Service, so it is only ever reachable in-cluster. + +Only the **north-south edge** (Ingress or Gateway) needs to block these paths and return +`403` for external callers. Two example manifests route the public API (`/`, +`/datasets/...`, etc.) to the `hsds` service while returning `403` for +`^/(metrics|info|about)$`: + +- ingress-nginx + Ingress: [`admin/kubernetes/k8s_ingress_nginx.yml`](../admin/kubernetes/k8s_ingress_nginx.yml) +- Gateway API + Envoy Gateway: [`admin/kubernetes/k8s_gateway_envoy.yml`](../admin/kubernetes/k8s_gateway_envoy.yml) + ## Suggested alerts ```yaml From 1fc19681d7fee3dcadd67f847785e98ad5a2cd00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Bastos?= Date: Mon, 27 Jul 2026 09:37:51 -0300 Subject: [PATCH 3/4] feat: instrument async internals --- docs/prometheus_metrics.md | 15 ++++ hsds/basenode.py | 3 +- hsds/chunk_crawl.py | 41 ++++++----- hsds/domain_crawl.py | 6 +- hsds/folder_crawl.py | 5 +- hsds/metrics.py | 142 +++++++++++++++++++++++++++++++------ hsds/util/httpUtil.py | 11 ++- tests/unit/metrics_test.py | 61 ++++++++++++++++ 8 files changed, 238 insertions(+), 46 deletions(-) diff --git a/docs/prometheus_metrics.md b/docs/prometheus_metrics.md index b50e65b5..12400c43 100644 --- a/docs/prometheus_metrics.md +++ b/docs/prometheus_metrics.md @@ -35,6 +35,15 @@ onto what actually degrades an HSDS deployment: - **Log-level counters** — `hsds_log_events_total{level="WARN"|"ERROR"}` is a cheap catch-all: alerting on ERROR rate catches problems that have no dedicated metric yet. +- **Async internals** — the HTTP middleware only sees top-level requests, so + three things it can't see are instrumented directly: + `hsds_internal_requests_total` / `hsds_internal_request_duration_seconds` + measure SN→DN fan-out (one client request can spawn hundreds of internal + calls), telling a slow DN apart from a slow SN; `hsds_crawler_queue_depth` / + `hsds_crawler_active_workers` expose crawler saturation (a growing queue with + workers pegged is overload *before* any 503); and + `hsds_housekeeping_last_success_timestamp_seconds` catches a stuck or dead + `healthCheck` loop. The default registry also provides `process_*` and `python_gc_*` metrics (CPU, RSS, FDs, GC) at no extra cost. @@ -59,6 +68,12 @@ The default registry also provides `process_*` and `python_gc_*` metrics | `hsds_storage_errors_total` | counter | `backend` | storage client errors | | `hsds_storage_bytes_read_total` | counter | `backend` | bytes read from storage | | `hsds_storage_bytes_written_total` | counter | `backend` | bytes written to storage | +| `hsds_internal_requests_total` | counter | `method`, `status` | internal SN→DN requests, via ClientSession TraceConfig | +| `hsds_internal_request_duration_seconds` | histogram | | internal SN→DN request latency | +| `hsds_crawler_queue_depth` | gauge | `crawler` (`chunk`/`domain`/`folder`) | items waiting in crawler queues | +| `hsds_crawler_active_workers` | gauge | `crawler` | crawler workers currently processing an item | +| `hsds_housekeeping_last_success_timestamp_seconds` | gauge | `task` | unix time a housekeeping task last succeeded | +| `hsds_housekeeping_duration_seconds` | histogram | `task` | housekeeping task run time | ## Scraping on Kubernetes diff --git a/hsds/basenode.py b/hsds/basenode.py index 7caf01a1..f83d759b 100644 --- a/hsds/basenode.py +++ b/hsds/basenode.py @@ -408,7 +408,8 @@ async def healthCheck(app): while True: try: - await doHealthCheck(app, chaos_die=chaos_die) + with metrics.housekeeping("health_check"): + await doHealthCheck(app, chaos_die=chaos_die) except Exception as e: msg = f"Unexpected {e.__class__.__name__} exception in " msg += f"doHealthCheck: {e}" diff --git a/hsds/chunk_crawl.py b/hsds/chunk_crawl.py index caa9f272..decb02fb 100755 --- a/hsds/chunk_crawl.py +++ b/hsds/chunk_crawl.py @@ -28,6 +28,7 @@ from .util.httpUtil import isUnixDomainUrl from .util.idUtil import getDataNodeUrl, getNodeCount from .util.hdf5dtype import createDataType +from . import metrics from .util.dsetUtil import getSliceQueryParam, getShapeDims from .util.dsetUtil import getSelectionShape, getChunkLayout from .util.chunkUtil import getChunkCoverage, getDataCoverage @@ -653,6 +654,7 @@ def __init__( for chunk_id in chunk_ids: self._q.put_nowait(chunk_id) + metrics.crawler_enqueued("chunk", len(chunk_ids)) self._bucket = bucket max_tasks = max_tasks_per_node * getNodeCount(app) @@ -719,27 +721,28 @@ async def work(self): try: start = time.time() chunk_id = await self._q.get() - if self._limit > 0 and self._hits >= self._limit: - msg = f"ChunkCrawler - maxhits exceeded, skipping fetch for chunk: {chunk_id}" - log.debug(msg) - else: - dn_url = getDataNodeUrl(self._app, chunk_id) - if isUnixDomainUrl(dn_url): - # need a client per url for unix sockets - client = get_http_client(self._app, url=dn_url, cache_client=True) + with metrics.crawler_task("chunk"): + if self._limit > 0 and self._hits >= self._limit: + msg = f"ChunkCrawler - maxhits exceeded, skipping fetch for chunk: {chunk_id}" + log.debug(msg) else: - # create a pool of clients and store the handles in the app dict - if client_name not in self._clients: - client = get_http_client( - self._app, url=dn_url, cache_client=False - ) - msg = "ChunkCrawler - creating new SessionClient for " - msg += f"task: {client_name}" - log.info(msg) - self._clients[client_name] = client + dn_url = getDataNodeUrl(self._app, chunk_id) + if isUnixDomainUrl(dn_url): + # need a client per url for unix sockets + client = get_http_client(self._app, url=dn_url, cache_client=True) else: - client = self._clients[client_name] - await self.do_work(chunk_id, client=client) + # create a pool of clients and store the handles in the app dict + if client_name not in self._clients: + client = get_http_client( + self._app, url=dn_url, cache_client=False + ) + msg = "ChunkCrawler - creating new SessionClient for " + msg += f"task: {client_name}" + log.info(msg) + self._clients[client_name] = client + else: + client = self._clients[client_name] + await self.do_work(chunk_id, client=client) self._q.task_done() elapsed = time.time() - start diff --git a/hsds/domain_crawl.py b/hsds/domain_crawl.py index b8e0ba39..92f48f35 100644 --- a/hsds/domain_crawl.py +++ b/hsds/domain_crawl.py @@ -23,6 +23,7 @@ from .util.globparser import globmatch from .servicenode_lib import getObjectJson, getAttributes, putAttributes, getLinks, putLinks from . import hsds_logger as log +from . import metrics class DomainCrawler: @@ -78,6 +79,7 @@ def __init__( for obj_id in objs: log.debug(f"adding {obj_id} to the queue") self._q.put_nowait(obj_id) + metrics.crawler_enqueued("domain", len(objs)) if isinstance(objs, dict): self._objs = objs else: @@ -124,6 +126,7 @@ def follow_links(self, grp_id, links): log.debug(f"DomainCrawler - adding link_id: {link_id} to queue") self._obj_dict[link_id] = {} # placeholder for obj id self._q.put_nowait(link_id) + metrics.crawler_enqueued("domain") link_count += 1 else: log.debug(f"link: {link_id} already in object dict") @@ -487,7 +490,8 @@ async def crawl(self): async def work(self): while True: obj_id = await self._q.get() - await self.fetch(obj_id) + with metrics.crawler_task("domain"): + await self.fetch(obj_id) self._q.task_done() async def fetch(self, obj_id): diff --git a/hsds/folder_crawl.py b/hsds/folder_crawl.py index 48f37ce6..1b10ea1a 100644 --- a/hsds/folder_crawl.py +++ b/hsds/folder_crawl.py @@ -22,6 +22,7 @@ from .util.idUtil import getNodeCount from .servicenode_lib import getObjectJson, getDomainResponse, getDomainJson from . import hsds_logger as log +from . import metrics class FolderCrawler: @@ -43,6 +44,7 @@ def __init__( self._group_dict = {} for domain in domains: self._q.put_nowait(domain) + metrics.crawler_enqueued("folder", len(domains)) self._bucket = bucket max_tasks = max_tasks_per_node * getNodeCount(app) if len(domains) > max_tasks: @@ -69,7 +71,8 @@ async def work(self): while True: start = time.time() domain = await self._q.get() - await self.fetch(domain) + with metrics.crawler_task("folder"): + await self.fetch(domain) self._q.task_done() elapsed = time.time() - start msg = f"FolderCrawler - task {domain} start: {start:.3f} " diff --git a/hsds/metrics.py b/hsds/metrics.py index b05c584e..7d5b6553 100644 --- a/hsds/metrics.py +++ b/hsds/metrics.py @@ -13,6 +13,8 @@ # Prometheus metrics exporter - serves /metrics in text exposition format # import asyncio +import contextlib +import time from aiohttp import web from prometheus_client import ( @@ -20,6 +22,7 @@ REGISTRY, CollectorRegistry, Counter, + Gauge, Histogram, generate_latest, ) @@ -34,35 +37,130 @@ _STORAGE_STATS = ("s3_stats", "azure_stats", "file_stats") -def _make_http_metrics(): - requests = Counter( - "hsds_http_requests", - "HTTP requests by method and status", - ["method", "status"], - ) - duration = Histogram( - "hsds_http_request_duration_seconds", - "HTTP request latency", - buckets=BUCKETS, - ) - return requests, duration - - -HTTP_REQUESTS, HTTP_DURATION = _make_http_metrics() +def _make_metrics(): + """create the counters/gauges/histograms held in the default registry""" + return { + "http_requests": Counter( + "hsds_http_requests", + "HTTP requests by method and status", + ["method", "status"], + ), + "http_duration": Histogram( + "hsds_http_request_duration_seconds", + "HTTP request latency", + buckets=BUCKETS, + ), + "internal_requests": Counter( + "hsds_internal_requests", + "Internal (SN->DN) requests by method and status", + ["method", "status"], + ), + "internal_duration": Histogram( + "hsds_internal_request_duration_seconds", + "Internal (SN->DN) request latency", + buckets=BUCKETS, + ), + "crawler_queue": Gauge( + "hsds_crawler_queue_depth", + "Items waiting in crawler queues", + ["crawler"], + ), + "crawler_workers": Gauge( + "hsds_crawler_active_workers", + "Crawler worker tasks currently processing an item", + ["crawler"], + ), + "housekeeping_last_success": Gauge( + "hsds_housekeeping_last_success_timestamp_seconds", + "Unix time a housekeeping task last completed successfully", + ["task"], + ), + "housekeeping_duration": Histogram( + "hsds_housekeeping_duration_seconds", + "Housekeeping task run time", + ["task"], + buckets=BUCKETS, + ), + } + + +_METRICS = _make_metrics() def reset(): - """re-create the http metrics with zeroed counts""" - global HTTP_REQUESTS, HTTP_DURATION - REGISTRY.unregister(HTTP_REQUESTS) - REGISTRY.unregister(HTTP_DURATION) - HTTP_REQUESTS, HTTP_DURATION = _make_http_metrics() + """re-create metrics with zeroed counts (used by tests)""" + global _METRICS + for collector in _METRICS.values(): + REGISTRY.unregister(collector) + _METRICS = _make_metrics() def observe_request(method, status, duration): """record one completed http request""" - HTTP_REQUESTS.labels(method=method, status=str(int(status))).inc() - HTTP_DURATION.observe(duration) + _METRICS["http_requests"].labels(method=method, status=str(int(status))).inc() + _METRICS["http_duration"].observe(duration) + + +def observe_internal_request(method, status, duration): + """record one completed internal SN->DN request""" + _METRICS["internal_requests"].labels(method=method, status=str(status)).inc() + _METRICS["internal_duration"].observe(duration) + + +def make_trace_config(): + """aiohttp TraceConfig that records outgoing (SN->DN) request metrics. + + Attach to every ClientSession so internal fan-out is measured without touching + each call site. + """ + from aiohttp import TraceConfig + + async def on_start(session, ctx, params): + ctx.start = asyncio.get_event_loop().time() + + async def on_end(session, ctx, params): + elapsed = asyncio.get_event_loop().time() - ctx.start + observe_internal_request(params.method, params.response.status, elapsed) + + async def on_exception(session, ctx, params): + elapsed = asyncio.get_event_loop().time() - ctx.start + observe_internal_request(params.method, "error", elapsed) + + trace_config = TraceConfig() + trace_config.on_request_start.append(on_start) + trace_config.on_request_end.append(on_end) + trace_config.on_request_exception.append(on_exception) + return trace_config + + +def crawler_enqueued(crawler, n=1): + """record n items added to a crawler's queue""" + _METRICS["crawler_queue"].labels(crawler=crawler).inc(n) + + +@contextlib.contextmanager +def crawler_task(crawler): + """wrap processing of one dequeued item: queue depth down, worker busy for the + duration""" + _METRICS["crawler_queue"].labels(crawler=crawler).dec() + workers = _METRICS["crawler_workers"].labels(crawler=crawler) + workers.inc() + try: + yield + finally: + workers.dec() + + +@contextlib.contextmanager +def housekeeping(task): + """time an out-of-band task; advance its last-success timestamp only on clean + completion (a stuck/failing loop then shows up as a stale timestamp)""" + start = time.time() + try: + yield + _METRICS["housekeeping_last_success"].labels(task=task).set(time.time()) + finally: + _METRICS["housekeeping_duration"].labels(task=task).observe(time.time() - start) @web.middleware diff --git a/hsds/util/httpUtil.py b/hsds/util/httpUtil.py index 0d43ae4a..a3b95825 100644 --- a/hsds/util/httpUtil.py +++ b/hsds/util/httpUtil.py @@ -28,6 +28,7 @@ from hsds.util.idUtil import isValidUuid from .. import hsds_logger as log +from .. import metrics from .. import config @@ -211,7 +212,10 @@ def get_http_client(app, url=None, cache_client=True): if socket_path: log.info(f"Initiating UnixConnector with path: {socket_path}") - client = ClientSession(connector=UnixConnector(path=socket_path)) + client = ClientSession( + connector=UnixConnector(path=socket_path), + trace_configs=[metrics.make_trace_config()], + ) if cache_client: socket_clients[socket_path] = client log.info(f"Socket Ready: {socket_path}") @@ -225,7 +229,10 @@ def get_http_client(app, url=None, cache_client=True): # read_buf_size = config.get("read_buf_size", default=10*1024*1024) # log.debug(f"setting read_buf_size to: {read_buf_size}") # kwargs['read_bufsize'] = read_buf_size - client = ClientSession(connector=TCPConnector(**kwargs)) + client = ClientSession( + connector=TCPConnector(**kwargs), + trace_configs=[metrics.make_trace_config()], + ) if cache_client: app["client"] = client diff --git a/tests/unit/metrics_test.py b/tests/unit/metrics_test.py index e95db73d..c9e21f3e 100644 --- a/tests/unit/metrics_test.py +++ b/tests/unit/metrics_test.py @@ -12,6 +12,7 @@ import asyncio import sys import unittest +from types import SimpleNamespace from aiohttp import web from aiohttp.test_utils import make_mocked_request @@ -127,6 +128,66 @@ async def err_handler(request): self.assertEqual(samples['hsds_http_requests_total{method="GET",status="204"}'], 1) self.assertEqual(samples['hsds_http_requests_total{method="PUT",status="503"}'], 1) + def testInternalRequestMetrics(self): + metrics.observe_internal_request("GET", 200, 0.05) + metrics.observe_internal_request("GET", 200, 0.2) + metrics.observe_internal_request("PUT", "error", 2.0) + samples = parse(metrics.render({})) + self.assertEqual(samples['hsds_internal_requests_total{method="GET",status="200"}'], 2) + self.assertEqual(samples['hsds_internal_requests_total{method="PUT",status="error"}'], 1) + self.assertEqual(samples["hsds_internal_request_duration_seconds_count"], 3) + self.assertAlmostEqual(samples["hsds_internal_request_duration_seconds_sum"], 2.25) + + def testTraceConfigRecordsInternalRequest(self): + async def run(): + tc = metrics.make_trace_config() + ctx = SimpleNamespace() + await tc.on_request_start[0](None, ctx, SimpleNamespace()) + params = SimpleNamespace( + method="GET", response=SimpleNamespace(status=200) + ) + await tc.on_request_end[0](None, ctx, params) + + asyncio.run(run()) + samples = parse(metrics.render({})) + self.assertEqual(samples['hsds_internal_requests_total{method="GET",status="200"}'], 1) + self.assertEqual(samples["hsds_internal_request_duration_seconds_count"], 1) + + def testCrawlerMetrics(self): + metrics.crawler_enqueued("chunk", 3) + with metrics.crawler_task("chunk"): + # one item in flight, two still waiting + samples = parse(metrics.render({})) + self.assertEqual(samples['hsds_crawler_active_workers{crawler="chunk"}'], 1) + self.assertEqual(samples['hsds_crawler_queue_depth{crawler="chunk"}'], 2) + samples = parse(metrics.render({})) + self.assertEqual(samples['hsds_crawler_active_workers{crawler="chunk"}'], 0) + self.assertEqual(samples['hsds_crawler_queue_depth{crawler="chunk"}'], 2) + + def testHousekeepingSuccess(self): + with metrics.housekeeping("health_check"): + pass + samples = parse(metrics.render({})) + self.assertEqual( + samples['hsds_housekeeping_duration_seconds_count{task="health_check"}'], 1 + ) + self.assertGreater( + samples['hsds_housekeeping_last_success_timestamp_seconds{task="health_check"}'], 0 + ) + + def testHousekeepingFailureLeavesLastSuccessUnset(self): + with self.assertRaises(ValueError): + with metrics.housekeeping("failing"): + raise ValueError("boom") + samples = parse(metrics.render({})) + # duration is still recorded, but last-success must NOT advance on failure + self.assertEqual( + samples['hsds_housekeeping_duration_seconds_count{task="failing"}'], 1 + ) + self.assertNotIn( + 'hsds_housekeeping_last_success_timestamp_seconds{task="failing"}', samples + ) + def testMetricsHandler(self): async def run(): app = {"node_state": "READY"} From 7f4d0a653725ce350de42ff27753177ae3aee155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Bastos?= Date: Tue, 28 Jul 2026 18:55:20 -0300 Subject: [PATCH 4/4] feat: Prometheus/Grafana monitoring stack, dashboard, and metric fixes - admin/docker/docker-compose.metrics.yml: Prometheus + Grafana overlay for any HSDS compose stack (Prometheus datasource pre-wired) - admin/prometheus/: example scrape config (relabels to the endpoint=sn|dn label the dashboard depends on) and promtool-checked alert rules - admin/grafana/hsds-dashboard.json: "HSDS" dashboard, 19 panels across cluster-internals and async/background-job rows - docs/prometheus_metrics.md + README: document the local stack, the endpoint-label dependency, and the dashboard; link the guide from the README - rename metric hsds_active_dn_count -> hsds_active_dns (the _count suffix is reserved for counters per Prometheus convention) - admin/kubernetes/k8s_deployment_aws.yml: declare the missing override configMap volume (kubectl apply would otherwise reject the manifest); drop a dead doc reference to a nonexistent singleton manifest Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 4 + admin/docker/docker-compose.metrics.yml | 37 + .../provisioning/datasources/datasource.yml | 7 + admin/grafana/hsds-dashboard.json | 2237 +++++++++++++++++ admin/kubernetes/k8s_deployment_aws.yml | 3 + admin/prometheus/alert.rules.yml | 42 + admin/prometheus/prometheus.yml | 33 + docs/kubernetes_install_aws.md | 3 - docs/prometheus_metrics.md | 69 +- hsds/metrics.py | 2 +- tests/unit/metrics_test.py | 2 +- 11 files changed, 2431 insertions(+), 8 deletions(-) create mode 100644 admin/docker/docker-compose.metrics.yml create mode 100644 admin/docker/grafana/provisioning/datasources/datasource.yml create mode 100644 admin/grafana/hsds-dashboard.json create mode 100644 admin/prometheus/alert.rules.yml create mode 100644 admin/prometheus/prometheus.yml diff --git a/README.md b/README.md index 586ddccf..3ca475fc 100755 --- a/README.md +++ b/README.md @@ -91,6 +91,10 @@ Authorization, ACLs, and Role Based Access Control (RBAC): - See [docs/authorization.md](docs/authorization.md) +Monitoring and metrics (Prometheus / Grafana): + +- See [docs/prometheus_metrics.md](docs/prometheus_metrics.md) + ## Writing Client Applications As a REST service, clients be developed using almost any programming language. The diff --git a/admin/docker/docker-compose.metrics.yml b/admin/docker/docker-compose.metrics.yml new file mode 100644 index 00000000..18953735 --- /dev/null +++ b/admin/docker/docker-compose.metrics.yml @@ -0,0 +1,37 @@ +# Local monitoring overlay: adds Prometheus + Grafana to any HSDS compose stack. +# +# ./build.sh --no-lint # build the HSDS image from this branch (has /metrics) +# docker compose \ +# -f admin/docker/docker-compose.posix.yml \ +# -f admin/docker/docker-compose.metrics.yml \ +# up -d --scale sn=1 --scale dn=2 +# +# Prometheus: http://localhost:9090 Grafana: http://localhost:3000 (anonymous admin) +# +# Grafana comes with the Prometheus datasource pre-wired. Import the dashboard by hand +# (Dashboards -> New -> Import, paste admin/grafana/hsds-dashboard.json): it is Grafana +# schema v2 and needs Grafana 12+, which does not support loading v2 via file provisioning. +services: + prometheus: + image: prom/prometheus:latest + restart: ${RESTART_POLICY} + ports: + - "9090:9090" + volumes: + - ${PWD}/admin/prometheus/:/etc/prometheus/ + command: + - --config.file=/etc/prometheus/prometheus.yml + + grafana: + image: grafana/grafana:latest + restart: ${RESTART_POLICY} + ports: + - "3000:3000" + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + volumes: + - ${PWD}/admin/docker/grafana/provisioning/:/etc/grafana/provisioning/ + depends_on: + - prometheus diff --git a/admin/docker/grafana/provisioning/datasources/datasource.yml b/admin/docker/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 00000000..0eddf262 --- /dev/null +++ b/admin/docker/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true diff --git a/admin/grafana/hsds-dashboard.json b/admin/grafana/hsds-dashboard.json new file mode 100644 index 00000000..1ef9e7a6 --- /dev/null +++ b/admin/grafana/hsds-dashboard.json @@ -0,0 +1,2237 @@ +{ + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "datasource": { + "name": "-- Grafana --" + }, + "group": "grafana", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + } + ], + "cursorSync": "Crosshair", + "description": "HSDS cluster internals and background jobs: node readiness, request rate/latency/errors, task-pool saturation, cache pressure, storage throughput, SN→DN fan-out, crawler queues, and housekeeping health.", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (endpoint) (hsds_node_ready)", + "instant": true, + "legendFormat": "{{endpoint}}", + "queryType": "instant", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Live node count by type. Both sn and dn should be non-zero; a drop to 0 on either means HSDS is down for that tier.", + "id": 1, + "links": [], + "title": "HSDS node readiness", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-10": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (endpoint, status) (rate(hsds_http_requests_total{status=~\"5..\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{endpoint}} {{status}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "503s mean load-shedding (task pool full) or DN unreachable. 5xx from the DN endpoint means storage backend errors are propagating up. Zero is the expected steady state — any rate here is actionable.", + "id": 10, + "links": [], + "title": "HSDS 5xx error rate by endpoint & status", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-11": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (level, endpoint) (rate(hsds_log_events_total[$__rate_interval]))", + "instant": false, + "legendFormat": "{{endpoint}} {{level}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "WARN and ERROR log lines per second, by node type. A rising ERROR rate is a catch-all signal for problems without a dedicated metric. Correlate spikes with the 5xx rate and cache dirty-item panels.", + "id": 11, + "links": [], + "title": "HSDS log events (WARN / ERROR rate)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*ERROR.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*WARN.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (backend) (rate(hsds_storage_bytes_read_total[$__rate_interval]))", + "instant": false, + "legendFormat": "{{backend}} read", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (backend) (rate(hsds_storage_bytes_written_total[$__rate_interval]))", + "instant": false, + "legendFormat": "{{backend}} write", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Bytes/s flowing through the HSDS storage backend. Read throughput spikes on cold-cache chunk fetches; write throughput reflects dirty-cache flushes. Will show No data until the first storage operation completes.", + "id": 12, + "links": [], + "title": "Storage throughput (read / written)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-13": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (backend) (rate(hsds_storage_errors_total[$__rate_interval]))", + "instant": false, + "legendFormat": "{{backend}} errors/s", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Storage client errors per second by backend. Any non-zero rate means HSDS is failing to read or write objects — correlate with rising request latency and dirty-cache growth. Zero is the expected steady state. Will show No data until the first storage operation completes.", + "id": 13, + "links": [], + "title": "Storage error rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0.001 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "avg(hsds_active_dns{endpoint=\"sn\"})", + "instant": true, + "legendFormat": "active DNs", + "queryType": "instant", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Data nodes visible to the service node. Should match expected DN replica count. A decrease means the SN lost contact with one or more DNs.", + "id": 2, + "links": [], + "title": "Active DN count (SN view)", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.50, sum by (le, endpoint) (rate(hsds_http_request_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "{{endpoint}} p50", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.95, sum by (le, endpoint) (rate(hsds_http_request_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "{{endpoint}} p95", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.99, sum by (le, endpoint) (rate(hsds_http_request_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "{{endpoint}} p99", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "HSDS request latency by node type (log scale). Compare tiers: high dn p95 with low sn p95 points at a slow data node or storage backend; high sn p95 means the service node itself is the bottleneck.", + "id": 3, + "links": [], + "title": "HSDS request latency p50/p95/p99", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (endpoint) (rate(hsds_http_requests_total[$__rate_interval]))", + "instant": false, + "legendFormat": "{{endpoint}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "HTTP req/s into service nodes (sn) and data nodes (dn). Sustained dn >> sn = many chunks per request (large dataset slices).", + "id": 4, + "links": [], + "title": "HSDS request rate by node type", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (version) (hsds_info{node_type=\"sn\"})", + "instant": true, + "legendFormat": "{{version}}", + "queryType": "instant", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Running HSDS version across all nodes. A mixed-version state during rollout shows multiple values here.", + "id": 5, + "links": [], + "title": "HSDS version", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(hsds_http_requests_total{status=~\"5..\"}[$__rate_interval])) or vector(0)", + "instant": true, + "legendFormat": "5xx/s", + "queryType": "instant", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Current 5xx response rate. 503 is the primary load-shedding signal: SNs return 503 when task slots are full or DNs are unreachable. Any sustained non-zero rate warrants investigation.", + "id": 6, + "links": [], + "title": "5xx error rate", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.001 + }, + { + "color": "red", + "value": 0.01 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (pod) (hsds_tasks_active) / sum by (pod) (hsds_tasks_max)", + "instant": false, + "legendFormat": "{{pod}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Fraction of the task pool consumed per pod. Task slots are the async I/O budget for the node. Saturation (>80%) means HSDS is queueing work and will slow down — this is the leading indicator for rising request latency and 503 load-shedding.", + "id": 7, + "links": [], + "title": "HSDS task pool utilization", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.95 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (cache) (hsds_cache_mem_used_bytes) / sum by (cache) (hsds_cache_mem_target_bytes)", + "instant": false, + "legendFormat": "{{cache}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Cache memory fill ratio per cache type (chunk / domain / meta). At 100% the cache evicts; evictions increase storage read pressure and can raise request latency. chunk cache matters most for repeated data slices.", + "id": 8, + "links": [], + "title": "HSDS cache memory utilization", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (cache) (hsds_cache_items)", + "instant": false, + "legendFormat": "{{cache}} items", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (cache) (hsds_cache_dirty_items)", + "instant": false, + "legendFormat": "{{cache}} dirty", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Cached item count (solid) and dirty items (dashed) by cache type. Dirty items are in-memory modifications pending flush to storage. A rising dirty count that doesn’t drain = stalled write pipeline. chunk cache dominates storage read savings; a low chunk item count suggests eviction pressure (cross-check the memory utilization panel).", + "id": 9, + "links": [], + "title": "HSDS cache items (total vs dirty)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-22": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "${datasource}" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (endpoint) (rate(hsds_internal_requests_total[$__rate_interval]))", + "instant": false, + "legendFormat": "{{endpoint}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Internal SN→DN request rate. Correlate with request rate by node type; sustained growth without matching upstream traffic indicates fan-out amplification.", + "id": 22, + "links": [], + "title": "SN→DN fan-out request rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "auto" }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-23": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "${datasource}" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.95, sum by (le) (rate(hsds_internal_request_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "p95", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 latency of internal SN→DN requests. High values here point to DN-side slowness contributing to overall request latency.", + "id": 23, + "links": [], + "title": "SN→DN fan-out latency (p95)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "auto" }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "${datasource}" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (crawler) (hsds_crawler_queue_depth)", + "instant": false, + "legendFormat": "{{crawler}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Pending items per crawler type (chunk/domain/folder). A growing queue means the crawler can't keep up with discovery work.", + "id": 24, + "links": [], + "title": "Crawler queue depth", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "auto" }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-25": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "${datasource}" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (crawler) (hsds_crawler_active_workers)", + "instant": false, + "legendFormat": "{{crawler}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Active worker count per crawler type. Zero workers with a non-zero queue depth means crawling has stalled.", + "id": 25, + "links": [], + "title": "Crawler active workers", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "auto" }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-26": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "${datasource}" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "time() - max by (task) (hsds_housekeeping_last_success_timestamp_seconds)", + "instant": false, + "legendFormat": "{{task}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Time elapsed since each housekeeping task last succeeded. A steadily rising value means that task has stopped completing — compare against its expected run interval.", + "id": 26, + "links": [], + "title": "Housekeeping: time since last success", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "auto" }, + "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 3600 } ] }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } + } + }, + "version": "13.2.0-29903742394" + } + } + }, + "panel-27": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "${datasource}" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (task) (rate(hsds_housekeeping_duration_seconds_sum[$__rate_interval])) / sum by (task) (rate(hsds_housekeeping_duration_seconds_count[$__rate_interval]))", + "instant": false, + "legendFormat": "{{task}}", + "queryType": "range", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Average run time of each housekeeping task (rate of _sum / rate of _count over the histogram). Rising duration trend can indicate growing dataset size or backend slowdown.", + "id": 27, + "links": [], + "title": "Housekeeping task duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "showPoints": "auto" }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } + } + }, + "version": "13.2.0-29903742394" + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "fillScreen": false, + "hideHeader": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-1" + }, + "height": 4, + "width": 6, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-2" + }, + "height": 4, + "width": 6, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-3" + }, + "height": 9, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-4" + }, + "height": 9, + "width": 12, + "x": 0, + "y": 4 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-5" + }, + "height": 4, + "width": 6, + "x": 12, + "y": 9 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-6" + }, + "height": 4, + "width": 6, + "x": 18, + "y": 9 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-7" + }, + "height": 9, + "width": 12, + "x": 0, + "y": 13 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-8" + }, + "height": 9, + "width": 12, + "x": 12, + "y": 13 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-9" + }, + "height": 9, + "width": 24, + "x": 0, + "y": 22 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-10" + }, + "height": 9, + "width": 12, + "x": 0, + "y": 31 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-11" + }, + "height": 9, + "width": 12, + "x": 12, + "y": 31 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-12" + }, + "height": 9, + "width": 12, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-13" + }, + "height": 9, + "width": 12, + "x": 12, + "y": 40 + } + } + ] + } + }, + "title": "HSDS internals" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "fillScreen": false, + "hideHeader": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-22" }, "height": 8, "width": 12, "x": 0, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-23" }, "height": 8, "width": 12, "x": 12, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-24" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-25" }, "height": 8, "width": 12, "x": 12, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-26" }, "height": 8, "width": 12, "x": 0, "y": 16 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-27" }, "height": 8, "width": 12, "x": 12, "y": 16 } } + ] + } + }, + "title": "Async / background jobs" + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preferences": { + "layout": { + "kind": "AutoGridLayout", + "spec": { + "columnWidthMode": "standard", + "items": [], + "maxColumnCount": 3, + "rowHeightMode": "standard" + } + } + }, + "preload": false, + "tags": ["hsds"], + "timeSettings": { + "autoRefresh": "1m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "fiscalYearStartMonth": 0, + "from": "now-1h", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "HSDS", + "variables": [ + { + "kind": "DatasourceVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": "dontHide", + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "pluginId": "prometheus", + "refresh": "onDashboardLoad", + "regex": "", + "skipUrlSync": false + } + } + ] +} diff --git a/admin/kubernetes/k8s_deployment_aws.yml b/admin/kubernetes/k8s_deployment_aws.yml index 8912af85..8ca6566f 100644 --- a/admin/kubernetes/k8s_deployment_aws.yml +++ b/admin/kubernetes/k8s_deployment_aws.yml @@ -115,3 +115,6 @@ spec: - name: config configMap: name: hsds-config + - name: override + configMap: + name: hsds-override diff --git a/admin/prometheus/alert.rules.yml b/admin/prometheus/alert.rules.yml new file mode 100644 index 00000000..d17f9193 --- /dev/null +++ b/admin/prometheus/alert.rules.yml @@ -0,0 +1,42 @@ +groups: + - name: hsds + rules: + - alert: HsdsNodeNotReady + expr: hsds_node_ready == 0 + for: 5m + - alert: HsdsHighErrorRate + expr: > + sum(rate(hsds_http_requests_total{status=~"5.."}[5m])) + / sum(rate(hsds_http_requests_total[5m])) > 0.05 + for: 10m + - alert: HsdsSlowRequests + expr: > + histogram_quantile(0.99, + rate(hsds_http_request_duration_seconds_bucket[5m])) > 5 + for: 10m + - alert: HsdsTaskSaturation + expr: hsds_tasks_active / hsds_tasks_max > 0.8 + for: 5m + - alert: HsdsDirtyCacheGrowing + expr: sum by (pod) (hsds_cache_dirty_items) > 100 + for: 15m + - alert: HsdsStorageErrors + expr: rate(hsds_storage_errors_total[5m]) > 0 + for: 5m + # SN->DN fan-out failing: a different plane than HsdsHighErrorRate (client + # facing) or HsdsStorageErrors (object store). Catches the SN losing contact + # with DNs, or DNs shedding load internally, before it fully becomes client 503s. + - alert: HsdsInternalErrors + expr: > + sum(rate(hsds_internal_requests_total{status=~"error|5.."}[5m])) + / sum(rate(hsds_internal_requests_total[5m])) > 0.05 + for: 10m + # healthCheck loop wedged/dead: the node still answers HTTP but has stopped + # reconciling DN membership. last_success only advances on a clean run, so a + # stuck loop shows up as a growing gap. The loop runs every node_sleep_time + # (default 10s), so >180s is ~18 missed cycles. The metric is absent until the + # first successful run (and in standalone mode, which has no health loop), so + # this complements HsdsNodeNotReady rather than replacing it. + - alert: HsdsHousekeepingStale + expr: time() - hsds_housekeeping_last_success_timestamp_seconds > 180 + for: 1m diff --git a/admin/prometheus/prometheus.yml b/admin/prometheus/prometheus.yml new file mode 100644 index 00000000..e162babb --- /dev/null +++ b/admin/prometheus/prometheus.yml @@ -0,0 +1,33 @@ +# Example Prometheus scrape config for a local docker-compose HSDS deployment. +# +# On Kubernetes, prefer a Prometheus Operator PodMonitor (see docs/prometheus_metrics.md) +# rather than this file. The important detail either way: each series is tagged with an +# `endpoint="sn"|"dn"` label so dashboards can split by node type. The PodMonitor derives +# that label from the port name (`- port: sn`); here we set it explicitly via relabeling. +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alert.rules.yml + +scrape_configs: + # SN containers. docker compose DNS returns one A record per replica, so a single + # dns_sd entry discovers all `--scale sn=N` replicas. + - job_name: hsds-sn + dns_sd_configs: + - names: [sn] + type: A + port: 5101 + relabel_configs: + - target_label: endpoint + replacement: sn + + - job_name: hsds-dn + dns_sd_configs: + - names: [dn] + type: A + port: 6101 + relabel_configs: + - target_label: endpoint + replacement: dn diff --git a/docs/kubernetes_install_aws.md b/docs/kubernetes_install_aws.md index f4cffeda..1d2016ec 100644 --- a/docs/kubernetes_install_aws.md +++ b/docs/kubernetes_install_aws.md @@ -130,9 +130,6 @@ If you need to build and deploy a custom HSDS image (e.g. you have made changes - image: '1234567.dkr.ecr.us-east-1.amazonaws.com/hsds:v1' to reflect the ecr repository for deployment - Note: if just one pod will be used, this deployment: **\*k8s_deployment_aws_singleton.yml\*\*** - can be used to provide multiple DN containers in one pod. - 5. Apply the deployment: `kubectl apply -f admin/kubernetes/k8s_deployment_aws.yml` 6. Verify that the HSDS pod is running: `kubectl get pods` a pod with a name starting with hsds should be displayed with status as "Running". 7. Additional verification: Run (`kubectl describe pod hsds-xxxx`) and make sure everything looks OK diff --git a/docs/prometheus_metrics.md b/docs/prometheus_metrics.md index 12400c43..3e5797e8 100644 --- a/docs/prometheus_metrics.md +++ b/docs/prometheus_metrics.md @@ -16,9 +16,9 @@ onto what actually degrades an HSDS deployment: on every request, including 4xx/5xx responses raised as exceptions. SN nodes return 503 when overloaded or when DNs are unreachable, so a rising `status="503"` rate is the primary "service is unhealthy" signal. -- **Cluster membership** — `hsds_node_ready` and `hsds_active_dn_count` +- **Cluster membership** — `hsds_node_ready` and `hsds_active_dns` detect nodes stuck outside `READY` and SNs that lost sight of DNs - (`hsds_active_dn_count` dropping below the expected DN count means reduced + (`hsds_active_dns` dropping below the expected DN count means reduced capacity even while requests still succeed). - **Saturation** — `hsds_tasks_active` vs `hsds_tasks_max`: when active asyncio tasks exceed `max_task_count`, SNs shed load with 503s. Watching @@ -55,7 +55,7 @@ The default registry also provides `process_*` and `python_gc_*` metrics | `hsds_info` | gauge | `node_type`, `node_id`, `version` | node identity, useful for version rollout tracking | | `hsds_node_ready` | gauge | | 1 when node state is `READY` | | `hsds_start_time_seconds` | gauge | | node start time (restart detection) | -| `hsds_active_dn_count` | gauge | | DN urls this node currently knows about | +| `hsds_active_dns` | gauge | | DN urls this node currently knows about | | `hsds_tasks_active` | gauge | | current asyncio task count | | `hsds_tasks_max` | gauge | | task count above which SNs return 503 | | `hsds_http_requests_total` | counter | `method`, `status` | all HTTP requests, via middleware | @@ -99,6 +99,37 @@ spec: path: /metrics ``` +Both discovery methods attach an `endpoint="sn"|"dn"` target label — the PodMonitor derives +it from the port name. The Grafana dashboard and several example queries split by that label, +so a scrape that does not set it (e.g. the bare `prometheus.io/scrape` annotation, which also +only reaches the SN) will leave those panels empty. + +## Local testing with Docker Compose + +`admin/docker/docker-compose.metrics.yml` adds Prometheus and Grafana to any HSDS compose +stack for local experimentation: + +```sh +./build.sh --no-lint # build the image from source (the published image may predate /metrics) +export ROOT_DIR=/tmp/hsds-data BUCKET_NAME=hsds.test HSDS_ENDPOINT=http://localhost:5101 +mkdir -p $ROOT_DIR/$BUCKET_NAME +docker compose -f admin/docker/docker-compose.posix.yml \ + -f admin/docker/docker-compose.metrics.yml up -d --scale sn=1 --scale dn=2 +``` + +Prometheus () scrapes both node types via +`admin/prometheus/prometheus.yml`, which relabels each job to the same `endpoint` label the +PodMonitor produces. Grafana (, anonymous admin) ships with the +Prometheus datasource pre-wired. + +## Grafana dashboard + +`admin/grafana/hsds-dashboard.json` is an "HSDS Internals" dashboard covering readiness, +request rate/latency/errors, task-pool saturation, cache pressure, and storage throughput. +It is exported in Grafana **schema v2** and requires **Grafana 12+**; import it through the UI +(Dashboards → New → Import), as v2 dashboards cannot be loaded via file provisioning. It relies +on the `endpoint` label described above. + ## Restricting external access to `/metrics` and `/info` `/metrics` and `/info` (and `/about`) expose operational detail — node topology, @@ -121,6 +152,9 @@ Only the **north-south edge** (Ingress or Gateway) needs to block these paths an ## Suggested alerts +These rules are also shipped as `admin/prometheus/alert.rules.yml` (validated with +`promtool check rules`): + ```yaml groups: - name: hsds @@ -147,4 +181,33 @@ groups: - alert: HsdsStorageErrors expr: rate(hsds_storage_errors_total[5m]) > 0 for: 5m + # SN->DN fan-out failing: a different plane than HsdsHighErrorRate (client + # facing) or HsdsStorageErrors (object store). Catches the SN losing contact + # with DNs, or DNs shedding load internally, before it fully becomes client 503s. + - alert: HsdsInternalErrors + expr: > + sum(rate(hsds_internal_requests_total{status=~"error|5.."}[5m])) + / sum(rate(hsds_internal_requests_total[5m])) > 0.05 + for: 10m + # healthCheck loop wedged/dead: the node still answers HTTP but has stopped + # reconciling DN membership. last_success only advances on a clean run, so a + # stuck loop shows up as a growing gap. The loop runs every node_sleep_time + # (default 10s), so >180s is ~18 missed cycles. The metric is absent until the + # first successful run (and in standalone mode, which has no health loop), so + # this complements HsdsNodeNotReady rather than replacing it. + - alert: HsdsHousekeepingStale + expr: time() - hsds_housekeeping_last_success_timestamp_seconds > 180 + for: 1m ``` + +The remaining new metrics are diagnostic (dashboard) rather than alerting signals: + +- `hsds_crawler_queue_depth` / `hsds_crawler_active_workers` — a deep queue is + normal for a large request (e.g. a selection spanning thousands of chunks), so a + threshold alert would be noisy. Sustained crawler saturation already surfaces via + `HsdsTaskSaturation` (crawler workers are asyncio tasks). Use these on a dashboard + to see *which* crawler is the bottleneck and whether workers are pegged at their + `max_tasks` ceiling. +- `hsds_internal_request_duration_seconds` — slow DNs already trip + `HsdsSlowRequests`; this histogram tells you *why* by separating internal fan-out + latency from SN-side processing time. diff --git a/hsds/metrics.py b/hsds/metrics.py index 7d5b6553..59b68332 100644 --- a/hsds/metrics.py +++ b/hsds/metrics.py @@ -215,7 +215,7 @@ def collect(self): if "dn_urls" in app: yield GaugeMetricFamily( - "hsds_active_dn_count", + "hsds_active_dns", "Data nodes this node currently knows about", value=len(app["dn_urls"]), ) diff --git a/tests/unit/metrics_test.py b/tests/unit/metrics_test.py index c9e21f3e..f6f39ef6 100644 --- a/tests/unit/metrics_test.py +++ b/tests/unit/metrics_test.py @@ -80,7 +80,7 @@ def testNodeMetrics(self): samples = parse(text) self.assertEqual(samples["hsds_node_ready"], 1) self.assertEqual(samples["hsds_start_time_seconds"], 1234567890) - self.assertEqual(samples["hsds_active_dn_count"], 2) + self.assertEqual(samples["hsds_active_dns"], 2) self.assertEqual(samples["hsds_tasks_max"], 100) self.assertIn('node_type="sn"', text) self.assertIn("hsds_tasks_active", text)