diff --git a/CHANGELOG.md b/CHANGELOG.md index c6dd4cd2..b2297ce0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,9 @@ ### Added -- Support listing OpenSearch Dashboards services ([#187](https://github.com/stackabletech/stackablectl/pull/187)). -- Add support for listing Spark History Servers ([#210](https://github.com/stackabletech/stackablectl/pull/210)). +- Support parametrization of stacks and demos ([#228](https://github.com/stackabletech/stackablectl/pull/228)) +- Support listing OpenSearch Dashboards services ([#187](https://github.com/stackabletech/stackablectl/pull/187)) +- Add support for listing Spark History Servers ([#210](https://github.com/stackabletech/stackablectl/pull/210)) ### Changed @@ -13,6 +14,7 @@ - Bump kube to 0.77 ([#201](https://github.com/stackabletech/stackablectl/pull/201)) - Bump go k8s client to 0.26.0 ([#202](https://github.com/stackabletech/stackablectl/pull/202)) - Bump kube, k8s-openapi and tokio ([#205](https://github.com/stackabletech/stackablectl/pull/205)) +- BREAKING: Bump format of demos and stacks to v2. Old versions of stackablectl will need an update ([#228](https://github.com/stackabletech/stackablectl/pull/228)) ## [0.6.0] - 2022-10-28 diff --git a/Cargo.lock b/Cargo.lock index 3b7910e4..a7b5bbb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,18 +61,78 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +[[package]] +name = "bcrypt" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7e7c93a3fb23b2fdde989b2c9ec4dd153063ec81f408507f84c090cd91c6641" +dependencies = [ + "base64 0.13.1", + "blowfish", + "getrandom", + "zeroize", +] + +[[package]] +name = "bcrypt" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9df288bec72232f78c1ec5fe4e8f1d108aa0265476e93097593c803c8c02062a" +dependencies = [ + "base64 0.21.0", + "blowfish", + "getrandom", + "subtle", + "zeroize", +] + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blowfish" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7" +dependencies = [ + "byteorder", + "cipher", +] + +[[package]] +name = "bstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f0778972c64420fdedc63f09919c8a88bda7b25135357fd25a5d9f3257e832" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "bumpalo" version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.4.0" @@ -142,6 +202,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "chrono-tz" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c39203181991a7dd4343b8005bd804e7a9a37afb8ac070e43771e8c820bbde" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f509c3a87b33437b05e2458750a0700e5bdd6956176773e6c7d6dd15a283a0c" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + +[[package]] +name = "cipher" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.1.4" @@ -226,6 +318,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + [[package]] name = "crossterm" version = "0.25.0" @@ -251,6 +352,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "cxx" version = "1.0.89" @@ -330,6 +441,22 @@ dependencies = [ "syn", ] +[[package]] +name = "deunicode" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "850878694b7933ca4c9569d30a34b55031b9b139ee1fc7b94a527c4ef960d690" + +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -528,6 +655,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.8" @@ -539,6 +676,30 @@ dependencies = [ "wasi", ] +[[package]] +name = "globset" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "globwalk" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" +dependencies = [ + "bitflags", + "ignore", + "walkdir", +] + [[package]] name = "gobuild" version = "0.1.0-alpha.2" @@ -640,6 +801,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humansize" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026" + [[package]] name = "humantime" version = "2.1.0" @@ -753,6 +920,23 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "ignore" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492" +dependencies = [ + "globset", + "lazy_static", + "log", + "memchr", + "regex", + "same-file", + "thread_local", + "walkdir", + "winapi-util", +] + [[package]] name = "indexmap" version = "1.9.2" @@ -764,6 +948,15 @@ dependencies = [ "serde", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -964,6 +1157,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.5.0" @@ -1134,6 +1333,28 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "parse-zoneinfo" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +dependencies = [ + "regex", +] + +[[package]] +name = "passwords" +version = "3.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d5e617847054b49f5b1ce93cea78a88fc269ef9210afdbca660c1e5c973212f" +dependencies = [ + "base64 0.13.1", + "bcrypt 0.13.0", + "md5", + "rand", + "random-pick", +] + [[package]] name = "pem" version = "1.1.1" @@ -1149,6 +1370,89 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +[[package]] +name = "pest" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ab62d2fa33726dbe6321cc97ef96d8cde531e3eeaf858a058de53a8a6d40d8f" +dependencies = [ + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf026e2d0581559db66d837fe5242320f525d85c76283c61f4d51a1238d65ea" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b27bd18aa01d91c8ed2b61ea23406a676b42d82609c6e2581fba42f0c15f17f" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02b677c1859756359fc9983c2e56a0237f18624a3789528804406b7e915e5d" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + +[[package]] +name = "phf" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +dependencies = [ + "siphasher", + "uncased", +] + [[package]] name = "pin-project" version = "1.0.12" @@ -1187,6 +1491,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1211,6 +1521,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.20+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" + [[package]] name = "proc-macro2" version = "1.0.51" @@ -1229,6 +1545,67 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "random-number" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3da5cbb4c27c5150c03a54a7e4745437cd90f9e329ae657c0b889a144bb7be" +dependencies = [ + "proc-macro-hack", + "rand", + "random-number-macro-impl", +] + +[[package]] +name = "random-number-macro-impl" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99f50024fe705be34d953ca47f4617ce3a665caa1011f14a48e6a8a6ec911f0f" +dependencies = [ + "proc-macro-hack", + "quote", + "syn", +] + +[[package]] +name = "random-pick" +version = "1.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c179499072da789afe44127d5f4aa6012de2c2f96ef759990196b37387a2a0f8" +dependencies = [ + "random-number", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -1338,6 +1715,15 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.21" @@ -1477,6 +1863,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "signal-hook" version = "0.3.14" @@ -1507,6 +1904,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "slab" version = "0.4.7" @@ -1516,6 +1919,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slug" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3bc762e6a4b6c6fcaade73e77f9ebc6991b676f88bb2358bddb56560f073373" +dependencies = [ + "deunicode", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -1536,6 +1948,7 @@ dependencies = [ name = "stackablectl" version = "0.6.0" dependencies = [ + "bcrypt 0.14.0", "cached", "clap", "clap_complete", @@ -1548,11 +1961,13 @@ dependencies = [ "lazy_static", "log", "openssl", + "passwords", "reqwest", "semver", "serde", "serde_json", "serde_yaml 0.9.17", + "tera", "tokio", "which", ] @@ -1582,6 +1997,12 @@ dependencies = [ "syn", ] +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "syn" version = "1.0.107" @@ -1607,6 +2028,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "tera" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df578c295f9ec044ff1c829daf31bb7581d5b3c2a7a3d87419afe1f2531438c" +dependencies = [ + "chrono", + "chrono-tz", + "globwalk", + "humansize", + "lazy_static", + "percent-encoding", + "pest", + "pest_derive", + "rand", + "regex", + "serde", + "serde_json", + "slug", + "unic-segment", +] + [[package]] name = "termcolor" version = "1.2.0" @@ -1636,6 +2079,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1815,6 +2267,77 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "ucd-trie" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" + +[[package]] +name = "uncased" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622" +dependencies = [ + "version_check", +] + +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +dependencies = [ + "unic-char-range", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" + +[[package]] +name = "unic-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" +dependencies = [ + "unic-ucd-segment", +] + +[[package]] +name = "unic-ucd-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +dependencies = [ + "unic-common", +] + [[package]] name = "unicode-bidi" version = "0.3.10" @@ -1871,6 +2394,17 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 7ec5d2fc..f4f975a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ license = "OSL-3.0" repository = "https://github.com/stackabletech/stackablectl" [dependencies] +bcrypt = "0.14" cached = { version = "0.42", features = ['async_tokio_rt_multi_thread'] } clap = { version = "4.1", features = ["derive", "cargo"] } clap_complete = "4.1" @@ -19,12 +20,14 @@ kube = { version = "0.78", default-features = false, features = ["client", "open lazy_static = "1.4" log = "0.4" openssl = { version = "0.10.45", features = ["vendored"] } # Must match version from kube +passwords = { version = "3.1", features = ["crypto"] } which = "4.4" semver = "1.0" serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" serde_yaml = "0.9" reqwest = "0.11" # Using native-tls as openssl does not seem to be supported as of 0.11 +tera = "1.17" tokio = "1.25" [profile.release] diff --git a/demos/demos-v1.yaml b/demos/demos-v1.yaml index b02fda5b..b2e4ee0f 100644 --- a/demos/demos-v1.yaml +++ b/demos/demos-v1.yaml @@ -1,140 +1,6 @@ ---- demos: - airflow-scheduled-job: - description: Activate a simple Airflow DAG to run continuously at a set interval - stackableStack: airflow - labels: - - airflow - - job-scheduling - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/01-airflow-spark-clusterrole.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/02-airflow-spark-clusterrolebinding.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml - hbase-hdfs-load-cycling-data: - description: Copy data from S3 bucket to an HBase table - stackableStack: hdfs-hbase - labels: - - hbase - - hdfs - - cycling-tripdata - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/hbase-hdfs-load-cycling-data/01-distcp-cycling-data.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/hbase-hdfs-load-cycling-data/02-create-hfile-and-import-to-hbase.yaml - nifi-kafka-druid-earthquake-data: - description: Demo ingesting earthquake data into Kafka using NiFi, streaming it into Druid and creating a Superset dashboard - documentation: https://docs.stackable.tech/stackablectl/stable/demos/nifi-kafka-druid-earthquake-data.html - stackableStack: nifi-kafka-druid-superset-s3 - labels: - - nifi - - kafka - - druid - - superset - - minio - - s3 - - earthquakes - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml - nifi-kafka-druid-water-level-data: - description: Demo ingesting water level data into Kafka using NiFi, streaming it into Druid and creating a Superset dashboard - documentation: https://docs.stackable.tech/stackablectl/stable/demos/nifi-kafka-druid-water-level-data.html - stackableStack: nifi-kafka-druid-superset-s3 - labels: - - nifi - - kafka - - druid - - superset - - minio - - s3 - - water-levels - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml - spark-k8s-anomaly-detection-taxi-data: - description: Demo loading New York taxi data into an S3 bucket and carrying out an anomaly detection analysis on it - documentation: https://docs.stackable.tech/stackablectl/stable/demos/spark-k8s-anomaly-detection-taxi-data.html - stackableStack: spark-trino-superset-s3 - labels: - - trino - - superset - - minio - - s3 - - ny-taxi-data - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/spark-k8s-anomaly-detection-taxi-data/serviceaccount.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml - trino-iceberg: - description: Demo containing Trino using Apache Iceberg as a S3 data lakehouse - documentation: https://docs.stackable.tech/stackablectl/stable/demos/trino-iceberg.html - stackableStack: trino-iceberg - labels: - - trino - - iceberg - - minio - - s3 + please-update: + description: This version of stackablectl is outdated, please visit https://docs.stackable.tech/stackablectl/stable/installation.html on how to get the latest version + stackableStack: does-not-exist + labels: [] manifests: [] - trino-taxi-data: - description: Demo loading 2.5 years of New York taxi data into S3 bucket, creating a Trino table and a Superset dashboard - documentation: https://docs.stackable.tech/stackablectl/stable/demos/trino-taxi-data.html - stackableStack: trino-superset-s3 - labels: - - trino - - superset - - minio - - s3 - - ny-taxi-data - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/trino-taxi-data/load-test-data.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/trino-taxi-data/create-table-in-trino.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/trino-taxi-data/setup-superset.yaml - data-lakehouse-iceberg-trino-spark: - description: Data lakehouse using Iceberg lakehouse on S3, Trino as query engine, Spark for streaming ingest and Superset for data visualization. Multiple datasources like taxi data, water levels in Germany, earthquakes, e-charging stations and more are loaded. - documentation: https://docs.stackable.tech/stackablectl/stable/demos/data-lakehouse-iceberg-trino-spark.html - stackableStack: data-lakehouse-iceberg-trino-spark - labels: - - iceberg - - trino - - spark - - superset - - kafka - - nifi - - minio - - s3 - - ny-taxi-data - - water-levels - - earthquakes - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml - jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data: - description: Jupyterhub with PySpark and HDFS integration - documentation: https://docs.stackable.tech/stackablectl/stable/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.html - stackableStack: jupyterhub-pyspark-hdfs - labels: - - jupyterhub - - hdfs - - pyspark - - ny-taxi-data - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml - logging: - description: Demo showing the logging stack in action - documentation: https://docs.stackable.tech/stackablectl/stable/demos/logging.html - stackableStack: logging - labels: - - logging - - opensearch - - opensearch-dashboards - - vector - - zookeeper - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/logging/zookeeper.yaml diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml new file mode 100644 index 00000000..826036f8 --- /dev/null +++ b/demos/demos-v2.yaml @@ -0,0 +1,140 @@ +--- +demos: + airflow-scheduled-job: + description: Activate a simple Airflow DAG to run continuously at a set interval + stackableStack: airflow + labels: + - airflow + - job-scheduling + manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/01-airflow-spark-clusterrole.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/02-airflow-spark-clusterrolebinding.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml + hbase-hdfs-load-cycling-data: + description: Copy data from S3 bucket to an HBase table + stackableStack: hdfs-hbase + labels: + - hbase + - hdfs + - cycling-tripdata + manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml + nifi-kafka-druid-earthquake-data: + description: Demo ingesting earthquake data into Kafka using NiFi, streaming it into Druid and creating a Superset dashboard + documentation: https://docs.stackable.tech/stackablectl/stable/demos/nifi-kafka-druid-earthquake-data.html + stackableStack: nifi-kafka-druid-superset-s3 + labels: + - nifi + - kafka + - druid + - superset + - minio + - s3 + - earthquakes + manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml + nifi-kafka-druid-water-level-data: + description: Demo ingesting water level data into Kafka using NiFi, streaming it into Druid and creating a Superset dashboard + documentation: https://docs.stackable.tech/stackablectl/stable/demos/nifi-kafka-druid-water-level-data.html + stackableStack: nifi-kafka-druid-superset-s3 + labels: + - nifi + - kafka + - druid + - superset + - minio + - s3 + - water-levels + manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml + spark-k8s-anomaly-detection-taxi-data: + description: Demo loading New York taxi data into an S3 bucket and carrying out an anomaly detection analysis on it + documentation: https://docs.stackable.tech/stackablectl/stable/demos/spark-k8s-anomaly-detection-taxi-data.html + stackableStack: spark-trino-superset-s3 + labels: + - trino + - superset + - minio + - s3 + - ny-taxi-data + manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/spark-k8s-anomaly-detection-taxi-data/serviceaccount.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml + trino-iceberg: + description: Demo containing Trino using Apache Iceberg as a S3 data lakehouse + documentation: https://docs.stackable.tech/stackablectl/stable/demos/trino-iceberg.html + stackableStack: trino-iceberg + labels: + - trino + - iceberg + - minio + - s3 + manifests: [] + trino-taxi-data: + description: Demo loading 2.5 years of New York taxi data into S3 bucket, creating a Trino table and a Superset dashboard + documentation: https://docs.stackable.tech/stackablectl/stable/demos/trino-taxi-data.html + stackableStack: trino-superset-s3 + labels: + - trino + - superset + - minio + - s3 + - ny-taxi-data + manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/trino-taxi-data/load-test-data.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/trino-taxi-data/create-table-in-trino.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/trino-taxi-data/setup-superset.yaml + data-lakehouse-iceberg-trino-spark: + description: Data lakehouse using Iceberg lakehouse on S3, Trino as query engine, Spark for streaming ingest and Superset for data visualization. Multiple datasources like taxi data, water levels in Germany, earthquakes, e-charging stations and more are loaded. + documentation: https://docs.stackable.tech/stackablectl/stable/demos/data-lakehouse-iceberg-trino-spark.html + stackableStack: data-lakehouse-iceberg-trino-spark + labels: + - iceberg + - trino + - spark + - superset + - kafka + - nifi + - minio + - s3 + - ny-taxi-data + - water-levels + - earthquakes + manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml + jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data: + description: Jupyterhub with PySpark and HDFS integration + documentation: https://docs.stackable.tech/stackablectl/stable/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.html + stackableStack: jupyterhub-pyspark-hdfs + labels: + - jupyterhub + - hdfs + - pyspark + - ny-taxi-data + manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml + logging: + description: Demo showing the logging stack in action + documentation: https://docs.stackable.tech/stackablectl/stable/demos/logging.html + stackableStack: logging + labels: + - logging + - opensearch + - opensearch-dashboards + - vector + - zookeeper + manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/logging/zookeeper.yaml diff --git a/src/demo.rs b/src/demo.rs index 6fc1a1af..3b984c67 100644 --- a/src/demo.rs +++ b/src/demo.rs @@ -1,7 +1,7 @@ use crate::{ arguments::OutputType, helpers, kind, - stack::{self, StackManifest}, + stack::{self, HelmChartOrYaml, StackParameter}, CliArgs, }; use cached::proc_macro::cached; @@ -18,7 +18,7 @@ use std::{error::Error, ops::Deref, sync::Mutex}; lazy_static! { pub static ref DEMO_FILES: Mutex> = Mutex::new(vec![ - "https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/demos-v1.yaml" + "https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/demos-v2.yaml" .to_string(), ]); } @@ -48,6 +48,20 @@ pub enum CliCommandDemo { #[arg(required = true, value_hint = ValueHint::Other)] demo: String, + /// List of parameters to use when installing the stack. + /// All parameters need to have the format `=`, e.g. `adminPassword=secret123`. + /// Multiple parameters can be specified. + /// Use `stackable stack describe ` to list the available parameters. + #[arg(short, long)] + stack_parameters: Vec, + + /// List of parameters to use when installing the demo. + /// All parameters need to have the format `=`, e.g. `adminPassword=secret123`. + /// Multiple parameters can be specified. + /// Use `stackable demo describe ` to list the available parameters. + #[arg(short, long)] + parameters: Vec, + /// If specified, a local Kubernetes cluster consisting of 4 nodes (1 for control-plane and 3 workers) for testing purposes will be created. /// Kind is a tool to spin up a local Kubernetes cluster running on Docker on your machine. /// You need to have `docker` and `kind` installed. @@ -73,11 +87,13 @@ impl CliCommandDemo { CliCommandDemo::Describe { demo, output } => describe_demo(demo, output).await?, CliCommandDemo::Install { demo, + stack_parameters, + parameters, kind_cluster, kind_cluster_name, } => { kind::handle_cli_arguments(*kind_cluster, kind_cluster_name)?; - install_demo(demo).await?; + install_demo(demo, stack_parameters, parameters).await?; } } Ok(()) @@ -102,8 +118,12 @@ struct Demo { description: String, documentation: Option, stackable_stack: String, + #[serde(default)] labels: Vec, - manifests: Vec, + #[serde(default)] + manifests: Vec, + #[serde(default)] + parameters: Vec, } async fn list_demos(output_type: &OutputType) -> Result<(), Box> { @@ -148,6 +168,7 @@ async fn describe_demo(demo_name: &str, output_type: &OutputType) -> Result<(), documentation: Option, stackable_stack: String, labels: Vec, + parameters: Vec, } let demo = get_demo(demo_name).await?; @@ -157,6 +178,7 @@ async fn describe_demo(demo_name: &str, output_type: &OutputType) -> Result<(), documentation: demo.documentation, stackable_stack: demo.stackable_stack, labels: demo.labels, + parameters: demo.parameters, }; match output_type { @@ -183,6 +205,24 @@ async fn describe_demo(demo_name: &str, output_type: &OutputType) -> Result<(), Cell::new(output.labels.join(", ")), ]); println!("{table}"); + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec![ + Cell::new("Parameter"), + Cell::new("Description"), + Cell::new("Default"), + ]); + for parameter in output.parameters { + table.add_row(vec![ + Cell::new(parameter.name), + Cell::new(parameter.description), + Cell::new(parameter.default), + ]); + } + println!("{table}"); } OutputType::Json => { println!("{}", serde_json::to_string_pretty(&output).unwrap()); @@ -195,12 +235,18 @@ async fn describe_demo(demo_name: &str, output_type: &OutputType) -> Result<(), Ok(()) } -async fn install_demo(demo_name: &str) -> Result<(), Box> { +async fn install_demo( + demo_name: &str, + stack_parameters: &[String], + parameters: &[String], +) -> Result<(), Box> { info!("Installing demo {demo_name}"); let demo = get_demo(demo_name).await?; - stack::install_stack(&demo.stackable_stack).await?; + let parameters = StackParameter::from_cli_parameters(&demo.parameters, parameters)?; + + stack::install_stack(&demo.stackable_stack, stack_parameters).await?; info!("Installing components of demo {demo_name}"); - stack::install_manifests(&demo.manifests).await?; + stack::install_manifests(&demo.manifests, ¶meters).await?; info!("Installed demo {demo_name}. Use \"stackablectl services list\" to list the installed services"); Ok(()) diff --git a/src/helpers.rs b/src/helpers.rs index f2300b60..ad29abe7 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -1,5 +1,6 @@ use log::trace; use std::{ + collections::HashMap, error::Error, ffi::CStr, fs, @@ -8,8 +9,11 @@ use std::{ process::{Command, Stdio}, str, }; +use tera::Context; use which::which; +use crate::templating::new_templating_instance; + #[repr(C)] pub struct GoString { p: *const u8, @@ -44,6 +48,24 @@ pub async fn read_from_url_or_file(url_or_file: &str) -> Result } } +pub async fn read_from_url_or_file_with_templating( + url_or_file: &str, + parameters: &HashMap, +) -> Result { + let file_content = read_from_url_or_file(url_or_file).await?; + + let mut context = Context::new(); + parameters.iter().for_each(|(name, value)| { + context.insert(name, value); + }); + + let mut tera = new_templating_instance() + .map_err(|err| format!("Failed to construct templating instance:: {err}"))?; + + tera.render_str(&file_content, &context) + .map_err(|err| format!("Failed to render template {url_or_file}: {err}")) +} + /// Ensures that the program is installed /// If the program is not installed it will return an Error pub fn ensure_program_installed(program: &str) -> Result<(), Box> { diff --git a/src/main.rs b/src/main.rs index 8d187977..8739cb1d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ mod operator; mod release; mod services; mod stack; +mod templating; const AVAILABLE_OPERATORS: &[&str] = &[ "airflow", diff --git a/src/services/grafana.rs b/src/services/grafana.rs new file mode 100644 index 00000000..99e45dd8 --- /dev/null +++ b/src/services/grafana.rs @@ -0,0 +1,61 @@ +use indexmap::IndexMap; +use k8s_openapi::api::core::v1::Service; +use kube::{api::ListParams, Api, ResourceExt}; +use std::{error::Error, vec}; + +use crate::{ + kube::{get_client, get_service_endpoint_urls}, + NAMESPACE, +}; + +use super::{get_credentials_from_secret, InstalledProduct}; + +pub async fn get_grafana_services( + namespaced: bool, + redact_credentials: bool, +) -> Result, Box> { + let client = get_client().await?; + let list_params = ListParams::default().labels("app.kubernetes.io/name=grafana"); + + let mut result = Vec::new(); + + let service_api: Api = match namespaced { + true => Api::namespaced(client.clone(), NAMESPACE.lock()?.as_str()), + false => Api::all(client.clone()), + }; + let services = service_api.list(&list_params).await?; + + for service in services { + let mut endpoints = IndexMap::new(); + let service_endpoint_urls = + get_service_endpoint_urls(&service, &service.name_unchecked(), client.clone()).await?; + endpoints.extend(service_endpoint_urls); + + let mut extra_infos = vec!["Third party service".to_string()]; + + // We assume the prom-operator was used for installation, in which case we know the secret name + let credentials = get_credentials_from_secret( + &service.name_unchecked(), + service + .namespace() + .as_ref() + .ok_or("Grafana service had no namespace set")?, + "admin-user", + "admin-password", + redact_credentials, + ) + .await?; + + if let Some((username, password)) = credentials { + extra_infos.push(format!("Admin user: {username}, password: {password}")); + } + + result.push(InstalledProduct { + name: service.name_unchecked(), + namespace: service.namespace(), + endpoints, + extra_infos, + }); + } + Ok(result) +} diff --git a/src/services/mod.rs b/src/services/mod.rs index e5d31150..68ef7e6c 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -11,11 +11,16 @@ use crate::{arguments::OutputType, kube::get_client}; use minio::get_minio_services; use opensearch::get_opensearch_dashboards_services; -use self::stackable::get_stackable_services; +use self::{ + grafana::get_grafana_services, prometheus::get_prometheus_services, + stackable::get_stackable_services, +}; -pub mod minio; -pub mod opensearch; -pub mod stackable; +mod grafana; +mod minio; +mod opensearch; +mod prometheus; +mod stackable; pub static REDACTED_PASSWORD: &str = ""; @@ -81,6 +86,16 @@ async fn list_services( output.insert("minio".to_string(), minio); } + let prometheus = get_grafana_services(!all_namespaces, redact_credentials).await?; + if !prometheus.is_empty() { + output.insert("grafana".to_string(), prometheus); + } + + let prometheus = get_prometheus_services(!all_namespaces, redact_credentials).await?; + if !prometheus.is_empty() { + output.insert("prometheus".to_string(), prometheus); + } + let opensearch = get_opensearch_dashboards_services(!all_namespaces, redact_credentials).await?; if !opensearch.is_empty() { diff --git a/src/services/prometheus.rs b/src/services/prometheus.rs new file mode 100644 index 00000000..642bf042 --- /dev/null +++ b/src/services/prometheus.rs @@ -0,0 +1,56 @@ +use indexmap::IndexMap; +use k8s_openapi::api::core::v1::Service; +use kube::{api::ListParams, Api, ResourceExt}; +use std::{error::Error, vec}; + +use crate::{ + kube::{get_client, get_service_endpoint_urls}, + NAMESPACE, +}; + +use super::InstalledProduct; + +const TRIM_SERVICE_NAME: &str = "-kube-prometheus-prometheus"; + +pub async fn get_prometheus_services( + namespaced: bool, + _redact_credentials: bool, +) -> Result, Box> { + let client = get_client().await?; + let list_params = ListParams::default() + .labels("app.kubernetes.io/instance=prometheus,app=kube-prometheus-stack-prometheus"); + + let mut result = Vec::new(); + + let service_api: Api = match namespaced { + true => Api::namespaced(client.clone(), NAMESPACE.lock()?.as_str()), + false => Api::all(client.clone()), + }; + let services = service_api.list(&list_params).await?; + + for service in services { + let prometheus_name = service + .name_unchecked() + .trim_end_matches(TRIM_SERVICE_NAME) + .to_string(); + + let mut endpoints = IndexMap::new(); + let service_endpoint_urls = get_service_endpoint_urls( + &service, + &service.name_unchecked(), // We pass the full service name (instead of prometheus_name) in here to not get long endpoint names + client.clone(), + ) + .await?; + endpoints.extend(service_endpoint_urls); + + let extra_infos = vec!["Third party service".to_string()]; + + result.push(InstalledProduct { + name: prometheus_name, + namespace: service.namespace(), + endpoints, + extra_infos, + }); + } + Ok(result) +} diff --git a/src/stack.rs b/src/stack.rs index 78b7a954..47f4202a 100644 --- a/src/stack.rs +++ b/src/stack.rs @@ -9,11 +9,11 @@ use indexmap::IndexMap; use lazy_static::lazy_static; use log::{debug, info, warn}; use serde::{Deserialize, Serialize}; -use std::{error::Error, ops::Deref, sync::Mutex}; +use std::{collections::HashMap, error::Error, ops::Deref, sync::Mutex}; lazy_static! { pub static ref STACK_FILES: Mutex> = Mutex::new(vec![ - "https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/stacks-v1.yaml" + "https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/stacks-v2.yaml" .to_string(), ]); } @@ -43,6 +43,13 @@ pub enum CliCommandStack { #[arg(required = true, value_hint = ValueHint::Other)] stack: String, + /// List of parameters to use when installing the stack. + /// All parameters need to have the format `=`, e.g. `adminPassword=secret123`. + /// Multiple parameters can be specified. + /// Use `stackable stack describe ` to list the available parameters. + #[arg(short, long)] + parameters: Vec, + /// If specified, a local Kubernetes cluster consisting of 4 nodes (1 for control-plane and 3 workers) for testing purposes will be created. /// Kind is a tool to spin up a local Kubernetes cluster running on Docker on your machine. /// You need to have `docker` and `kind` installed. @@ -68,11 +75,12 @@ impl CliCommandStack { CliCommandStack::Describe { stack, output } => describe_stack(stack, output).await?, CliCommandStack::Install { stack, + parameters, kind_cluster, kind_cluster_name, } => { kind::handle_cli_arguments(*kind_cluster, kind_cluster_name)?; - install_stack(stack).await?; + install_stack(stack, parameters).await?; } } Ok(()) @@ -96,24 +104,32 @@ struct Stacks { struct Stack { description: String, stackable_release: String, + stackable_operators: Vec, + #[serde(default)] labels: Vec, - manifests: Vec, + #[serde(default)] + manifests: Vec, + #[serde(default)] + parameters: Vec, } #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -pub enum StackManifest { - #[serde(rename_all = "camelCase")] - HelmChart { - release_name: String, - name: String, - repo: HelmChartRepo, - version: String, - options: serde_yaml::Value, - }, +pub enum HelmChartOrYaml { + HelmChart(String), PlainYaml(String), } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HelmChart { + release_name: String, + name: String, + repo: HelmChartRepo, + version: String, + options: serde_yaml::Value, +} + #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct HelmChartRepo { @@ -121,6 +137,49 @@ pub struct HelmChartRepo { url: String, } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StackParameter { + pub name: String, + pub description: String, + pub default: String, +} + +impl StackParameter { + const PARSE_PARAMETER_ERROR_MESSAGE: &str = "Can not parse parameter. Parameters need to have the format `=`, e.g. `adminPassword=secret123`."; + + pub fn from_cli_parameters( + stack_parameters: &[StackParameter], + cli_parameters: &[String], + ) -> Result, Box> { + let mut parameters: HashMap = stack_parameters + .iter() + .map(|p| (p.name.clone(), p.default.clone())) + .collect(); + + for parameter in cli_parameters { + let mut split = parameter.split('='); + let name = split.next().ok_or(Self::PARSE_PARAMETER_ERROR_MESSAGE)?; + let value = split.next().ok_or(Self::PARSE_PARAMETER_ERROR_MESSAGE)?; + + if !parameters.contains_key(name) { + return Err(format!( + "Parameter {name} not known. Known parameters are {}", + stack_parameters + .iter() + .map(|p| p.name.clone()) + .collect::>() + .join(", ") + ) + .into()); + } + parameters.insert(name.to_string(), value.to_string()); + } + + Ok(parameters) + } +} + async fn list_stacks(output_type: &OutputType) -> Result<(), Box> { let output = get_stacks().await; match output_type { @@ -161,7 +220,9 @@ async fn describe_stack(stack_name: &str, output_type: &OutputType) -> Result<() stack: String, description: String, stackable_release: String, + stackable_operators: Vec, labels: Vec, + parameters: Vec, } let stack = get_stack(stack_name).await?; @@ -169,7 +230,9 @@ async fn describe_stack(stack_name: &str, output_type: &OutputType) -> Result<() stack: stack_name.to_string(), description: stack.description, stackable_release: stack.stackable_release, + stackable_operators: stack.stackable_operators, labels: stack.labels, + parameters: stack.parameters, }; match output_type { @@ -187,11 +250,33 @@ async fn describe_stack(stack_name: &str, output_type: &OutputType) -> Result<() Cell::new("Stackable release"), Cell::new(output.stackable_release), ]) + .add_row(vec![ + Cell::new("Stackable operators"), + Cell::new(output.stackable_operators.join(", ")), + ]) .add_row(vec![ Cell::new("Labels"), Cell::new(output.labels.join(", ")), ]); println!("{table}"); + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec![ + Cell::new("Parameter"), + Cell::new("Description"), + Cell::new("Default"), + ]); + for parameter in output.parameters { + table.add_row(vec![ + Cell::new(parameter.name), + Cell::new(parameter.description), + Cell::new(parameter.default), + ]); + } + println!("{table}"); } OutputType::Json => { println!("{}", serde_json::to_string_pretty(&output)?); @@ -204,53 +289,64 @@ async fn describe_stack(stack_name: &str, output_type: &OutputType) -> Result<() Ok(()) } -pub async fn install_stack(stack_name: &str) -> Result<(), Box> { +pub async fn install_stack(stack_name: &str, parameters: &[String]) -> Result<(), Box> { info!("Installing stack {stack_name}"); let stack = get_stack(stack_name).await?; + let parameters = StackParameter::from_cli_parameters(&stack.parameters, parameters)?; - release::install_release(&stack.stackable_release, &[], &[]).await?; + release::install_release(&stack.stackable_release, &stack.stackable_operators, &[]).await?; info!("Installing components of stack {stack_name}"); - install_manifests(&stack.manifests).await?; + install_manifests(&stack.manifests, ¶meters).await?; info!("Installed stack {stack_name}"); Ok(()) } -pub async fn install_manifests(manifests: &[StackManifest]) -> Result<(), Box> { +pub async fn install_manifests( + manifests: &[HelmChartOrYaml], + parameters: &HashMap, +) -> Result<(), Box> { for manifest in manifests { match manifest { - StackManifest::HelmChart { - release_name, - name, - repo, - version, - options, - } => { - debug!("Installing helm chart {name} as {release_name}"); + HelmChartOrYaml::HelmChart(helm_chart_file) => { + let helm_chart = + helpers::read_from_url_or_file_with_templating(helm_chart_file, parameters) + .await + .map_err(|err| { + format!( + "Could not read helm chart from file \"{helm_chart_file}\": {err}" + ) + })?; + let helm_chart: HelmChart = serde_yaml::from_str(&helm_chart)?; + debug!( + "Installing helm chart {} as {}", + helm_chart.name, helm_chart.release_name + ); HELM_REPOS .lock()? - .insert(repo.name.clone(), repo.url.clone()); + .insert(helm_chart.repo.name.clone(), helm_chart.repo.url.clone()); - let values_yaml = serde_yaml::to_string(&options)?; + let values_yaml = serde_yaml::to_string(&helm_chart.options)?; helm::install_helm_release_from_repo( - release_name, - release_name, - &repo.name, - name, - Some(version), + &helm_chart.release_name, + &helm_chart.release_name, + &helm_chart.repo.name, + &helm_chart.name, + Some(&helm_chart.version), Some(&values_yaml), )? } - StackManifest::PlainYaml(yaml_url_or_file) => { + HelmChartOrYaml::PlainYaml(yaml_url_or_file) => { debug!("Installing yaml manifest from {yaml_url_or_file}"); - let manifests = helpers::read_from_url_or_file(yaml_url_or_file) - .await - .map_err(|err| { - format!( + let manifests = + helpers::read_from_url_or_file_with_templating(yaml_url_or_file, parameters) + .await + .map_err(|err| { + format!( "Could not read stack manifests from file \"{yaml_url_or_file}\": {err}" ) - })?; + })?; kube::deploy_manifests(&manifests).await?; } } diff --git a/src/templating.rs b/src/templating.rs new file mode 100644 index 00000000..e1364fd2 --- /dev/null +++ b/src/templating.rs @@ -0,0 +1,47 @@ +use std::{collections::HashMap, error::Error}; + +use bcrypt::{hash, DEFAULT_COST}; +use passwords::PasswordGenerator; +use tera::{from_value, Function, Tera, Value}; + +const PASSWORD_LEN: usize = 30; + +pub fn new_templating_instance() -> Result> { + // tera requires a glob of templates files to parse to offer better performance. + // As we only template the files once there is no benefit to collect the needed files and pass it in here. + // We just give some non-existent file and pass the filename to template during templating itself. + let mut tera = Tera::new("this-folder-does-not-exist/*.yaml")?; + tera.register_function("random_password", random_password()); + tera.register_function("bcrypt", bcrypt()); + Ok(tera) +} + +fn random_password() -> impl Function { + Box::new( + move |_args: &HashMap| -> tera::Result { + let password = PasswordGenerator::new() + .length(PASSWORD_LEN) + .generate_one() + .map_err(|err| format!("Failed to generate password: {err}"))?; + Ok(password.into()) + }, + ) +} + +fn bcrypt() -> impl Function { + Box::new( + move |args: &HashMap| -> tera::Result { + match args.get("password") { + Some(val) => match from_value::(val.clone()) { + Ok(password) => { + let hash = hash(password, DEFAULT_COST) + .map_err(|err| format!("Failed to create bcrypt hash: {err}"))?; + Ok(hash.into()) + } + Err(_) => Err("Cant get value of password".into()), + }, + None => Err("Parameter password missing".into()), + } + }, + ) +} diff --git a/stacks/_templates/jupyterhub.yaml b/stacks/_templates/jupyterhub.yaml new file mode 100644 index 00000000..4e1309a5 --- /dev/null +++ b/stacks/_templates/jupyterhub.yaml @@ -0,0 +1,55 @@ +releaseName: jupyterhub +name: jupyterhub +repo: + name: jupyterhub + url: https://jupyterhub.github.io/helm-chart/ +version: 2.0.0 +options: + proxy: + service: + type: NodePort + hub: + config: + Authenticator: + allowed_users: + - admin + DummyAuthenticator: + password: adminadmin + JupyterHub: + authenticator_class: dummy + rbac: + create: true + singleuser: + serviceAccountName: spark + extraEnv: + HADOOP_CONF_DIR: "/home/jovyan/hdfs" + initContainers: + - name: download-notebook + image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.4.0 + command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb -o /notebook/notebook.ipynb'] + volumeMounts: + - mountPath: /notebook + name: notebook + storage: + extraVolumes: + - name: hdfs-discovery-configmap + configMap: + name: hdfs + - name: notebook + emptyDir: + sizeLimit: 500Mi + extraVolumeMounts: + - name: hdfs-discovery-configmap + mountPath: /home/jovyan/hdfs + - name: notebook + mountPath: /home/jovyan/notebook + image: + # You should replace the "latest" tag with a fixed version from: + # https://hub.docker.com/r/jupyter/datascience-notebook/tags/ + # Inspect the Dockerfile at: + # https://github.com/jupyter/docker-stacks/tree/HEAD/datascience-notebook/Dockerfile + name: jupyter/pyspark-notebook + tag: python-3.9 + # `cmd: null` allows the custom CMD of the Jupyter docker-stacks to be used + # which performs further customization on startup. + cmd: null diff --git a/stacks/_templates/minio-distributed-small.yaml b/stacks/_templates/minio-distributed-small.yaml new file mode 100644 index 00000000..c56315a9 --- /dev/null +++ b/stacks/_templates/minio-distributed-small.yaml @@ -0,0 +1,30 @@ +releaseName: minio +name: minio +repo: + name: minio + url: https://charts.min.io/ +version: 4.0.15 +options: + rootUser: admin + rootPassword: adminadmin + mode: distributed + replicas: 2 + persistence: + size: 50Gi + users: + - accessKey: minio + secretKey: miniominio + policy: readwrite + buckets: + - name: lakehouse + policy: public + resources: + requests: + cpu: 500m + memory: 2Gi + service: + type: NodePort + nodePort: null + consoleService: + type: NodePort + nodePort: null diff --git a/stacks/_templates/minio-distributed.yaml b/stacks/_templates/minio-distributed.yaml new file mode 100644 index 00000000..60f5f0ca --- /dev/null +++ b/stacks/_templates/minio-distributed.yaml @@ -0,0 +1,38 @@ +releaseName: minio +name: minio +repo: + name: minio + url: https://charts.min.io/ +version: 4.0.15 +options: + rootUser: admin + rootPassword: adminadmin + mode: distributed + replicas: 5 + persistence: + size: 250Gi + users: + - accessKey: trino + secretKey: trinotrino + policy: readwrite + - accessKey: hive + secretKey: hivehive + policy: readwrite + - accessKey: demo + secretKey: demodemo + policy: readwrite + buckets: + - name: staging + policy: public + - name: lakehouse + policy: public + resources: + requests: + cpu: 500m + memory: 2Gi + service: + type: NodePort + nodePort: null + consoleService: + type: NodePort + nodePort: null diff --git a/stacks/_templates/minio-druid.yaml b/stacks/_templates/minio-druid.yaml new file mode 100644 index 00000000..a4707be8 --- /dev/null +++ b/stacks/_templates/minio-druid.yaml @@ -0,0 +1,28 @@ +releaseName: minio-druid +name: minio +repo: + name: minio + url: https://charts.min.io/ +version: 4.0.15 +options: + rootUser: admin + rootPassword: adminadmin + mode: standalone + persistence: + size: 15Gi + users: + - accessKey: druid + secretKey: druiddruid + policy: readwrite + buckets: + - name: druid + policy: public + resources: + requests: + memory: 2Gi + service: + type: NodePort + nodePort: null + consoleService: + type: NodePort + nodePort: null diff --git a/stacks/_templates/minio-trino.yaml b/stacks/_templates/minio-trino.yaml new file mode 100644 index 00000000..dcf24b39 --- /dev/null +++ b/stacks/_templates/minio-trino.yaml @@ -0,0 +1,35 @@ +releaseName: minio-trino +name: minio +repo: + name: minio + url: https://charts.min.io/ +version: 4.0.15 +options: + rootUser: admin + rootPassword: adminadmin + mode: standalone + persistence: + size: 10Gi + users: + - accessKey: trino + secretKey: trinotrino + policy: readwrite + - accessKey: hive + secretKey: hivehive + policy: readwrite + - accessKey: demo + secretKey: demodemo + policy: readwrite + buckets: + - name: demo + policy: public + resources: + requests: + cpu: 500m + memory: 1Gi + service: + type: NodePort + nodePort: null + consoleService: + type: NodePort + nodePort: null diff --git a/stacks/_templates/opensearch-dashboards.yaml b/stacks/_templates/opensearch-dashboards.yaml new file mode 100644 index 00000000..16f67a5a --- /dev/null +++ b/stacks/_templates/opensearch-dashboards.yaml @@ -0,0 +1,65 @@ +releaseName: opensearch-dashboards +name: opensearch-dashboards +repo: + name: opensearch-dashboards + url: https://opensearch-project.github.io/helm-charts +version: 2.7.0 +options: + service: + type: NodePort + port: 5601 + annotations: + stackable.tech/logging-view-logs: |- + /app/discover?security_tenant=global#/view/logs + stackable.tech/logging-credentials-secret: opensearch-user + opensearchAccount: + secret: opensearch-dashboard-user + extraEnvs: + - name: OPEN_SEARCH_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: opensearch-user + key: password + lifecycle: + postStart: + exec: + command: + - bash + - -c + - | + #!/bin/bash + curl \ + --retry-connrefused \ + --retry 120 \ + --retry-delay 5 \ + --header "Content-Type:application/json" \ + --header "osd-xsrf:true" \ + --header "securitytenant: global" \ + --data '{ + "attributes": { + "title": "vector-*", + "timeFieldName": "timestamp" + } + }' \ + "http://admin:${OPEN_SEARCH_ADMIN_PASSWORD}@127.0.0.1:5601/api/saved_objects/index-pattern/vector-*" + curl \ + --header "Content-Type:application/json" \ + --header "osd-xsrf:true" \ + --header "securitytenant: global" \ + --data '{ + "attributes": { + "title":"Logs", + "columns": ["pod", "container", "logger", "level", "message"], + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"indexRefName\": \"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "id": "vector-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ] + }' \ + http://admin:${OPEN_SEARCH_ADMIN_PASSWORD}@127.0.0.1:5601/api/saved_objects/search/logs diff --git a/stacks/_templates/opensearch.yaml b/stacks/_templates/opensearch.yaml new file mode 100644 index 00000000..c958066c --- /dev/null +++ b/stacks/_templates/opensearch.yaml @@ -0,0 +1,107 @@ +releaseName: opensearch +name: opensearch +repo: + name: opensearch + url: https://opensearch-project.github.io/helm-charts +version: 2.9.0 +options: + config: + opensearch.yml: | + plugins: + security: + # Use default security settings + allow_default_init_securityindex: true + # Allow communication between the nodes which use the + # certificates generated by the secret-operator + nodes_dn: + - CN=generated certificate for pod + # Use the certificate generated by the secret-operator + ssl: + http: + # Enable TLS on the REST layer + enabled: true + pemcert_filepath: certs/tls.crt + pemkey_filepath: certs/tls.key + pemtrustedcas_filepath: certs/ca.crt + transport: + pemcert_filepath: certs/tls.crt + pemkey_filepath: certs/tls.key + pemtrustedcas_filepath: certs/ca.crt + # Disable the verification of hostnames because + # internal IPs are used which are not included in + # the certificates generated by the secret-operator. + enforce_hostname_verification: false + securityConfig: + path: /usr/share/opensearch/config/opensearch-security + internalUsersSecret: opensearch-internal-users + sysctlInit: + enabled: {{ setSysctlMaxMapCount }} + extraEnvs: + # Disable the creation of demo certificates + - name: DISABLE_INSTALL_DEMO_CONFIG + value: "true" + extraVolumeMounts: + # Mount the certificate generated by the secret-operator + - name: tls + mountPath: /usr/share/opensearch/config/certs + extraVolumes: + # Request a TLS certificate from the secret-operator + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + # Add the service opensearch-cluster-master to the + # distinguished names because this service is used + # by Vector. + secrets.stackable.tech/scope: |- + service=opensearch-cluster-master + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + extraObjects: + - apiVersion: v1 + kind: Secret + metadata: + name: opensearch-internal-users + stringData: + internal_users.yml: | + --- + # This is the internal user database. + # The hash value is a bcrypt hash. + + _meta: + type: "internalusers" + config_version: 2 + + admin: + hash: "{{ bcrypt(password=openSearchAdminPassword) }}" + reserved: true + backend_roles: + - "admin" + description: "OpenSearch admin user" + + kibanaserver: + hash: "{{ bcrypt(password=openSearchDashboardPassword) }}" + reserved: true + description: "OpenSearch Dashboard user" + - apiVersion: v1 + kind: Secret + metadata: + name: opensearch-user + stringData: + username: admin + password: {{ openSearchAdminPassword }} + - apiVersion: v1 + kind: Secret + metadata: + name: opensearch-dashboard-user + stringData: + username: kibanaserver + password: {{ openSearchDashboardPassword }} + cookie: {{ random_password() }} diff --git a/stacks/_templates/postgresql-airflow.yaml b/stacks/_templates/postgresql-airflow.yaml new file mode 100644 index 00000000..4721a88e --- /dev/null +++ b/stacks/_templates/postgresql-airflow.yaml @@ -0,0 +1,11 @@ +releaseName: postgresql-airflow +name: postgresql +repo: + name: bitnami + url: https://charts.bitnami.com/bitnami/ +version: 12.1.5 +options: + auth: + username: airflow + password: airflow + database: airflow diff --git a/stacks/_templates/postgresql-druid.yaml b/stacks/_templates/postgresql-druid.yaml new file mode 100644 index 00000000..502c8b00 --- /dev/null +++ b/stacks/_templates/postgresql-druid.yaml @@ -0,0 +1,11 @@ +releaseName: postgresql-druid +name: postgresql +repo: + name: bitnami + url: https://charts.bitnami.com/bitnami/ +version: 12.1.5 +options: + auth: + username: druid + password: druid + database: druid diff --git a/stacks/_templates/postgresql-hive-iceberg.yaml b/stacks/_templates/postgresql-hive-iceberg.yaml new file mode 100644 index 00000000..88e38c93 --- /dev/null +++ b/stacks/_templates/postgresql-hive-iceberg.yaml @@ -0,0 +1,13 @@ +releaseName: postgresql-hive-iceberg +name: postgresql +repo: + name: bitnami + url: https://charts.bitnami.com/bitnami/ +version: 12.1.5 +options: + auth: + username: hive + password: hive + database: hive + primary: + extendedConfiguration: password_encryption=md5 diff --git a/stacks/_templates/postgresql-hive.yaml b/stacks/_templates/postgresql-hive.yaml new file mode 100644 index 00000000..48911846 --- /dev/null +++ b/stacks/_templates/postgresql-hive.yaml @@ -0,0 +1,13 @@ +releaseName: postgresql-hive +name: postgresql +repo: + name: bitnami + url: https://charts.bitnami.com/bitnami/ +version: 12.1.5 +options: + auth: + username: hive + password: hive + database: hive + primary: + extendedConfiguration: password_encryption=md5 diff --git a/stacks/_templates/postgresql-hivehdfs.yaml b/stacks/_templates/postgresql-hivehdfs.yaml new file mode 100644 index 00000000..becd1f1c --- /dev/null +++ b/stacks/_templates/postgresql-hivehdfs.yaml @@ -0,0 +1,13 @@ +releaseName: hivehdfs +name: postgresql +repo: + name: bitnami + url: https://charts.bitnami.com/bitnami/ +version: 12.1.5 +options: + auth: + username: hive + password: hive + database: hivehdfs + primary: + extendedConfiguration: password_encryption=md5 diff --git a/stacks/_templates/postgresql-hives3.yaml b/stacks/_templates/postgresql-hives3.yaml new file mode 100644 index 00000000..72f52d76 --- /dev/null +++ b/stacks/_templates/postgresql-hives3.yaml @@ -0,0 +1,13 @@ +releaseName: hives3 +name: postgresql +repo: + name: bitnami + url: https://charts.bitnami.com/bitnami/ +version: 12.1.5 +options: + auth: + username: hive + password: hive + database: hives3 + primary: + extendedConfiguration: password_encryption=md5 diff --git a/stacks/_templates/postgresql-superset.yaml b/stacks/_templates/postgresql-superset.yaml new file mode 100644 index 00000000..fdeceefd --- /dev/null +++ b/stacks/_templates/postgresql-superset.yaml @@ -0,0 +1,11 @@ +releaseName: postgresql-superset +name: postgresql +repo: + name: bitnami + url: https://charts.bitnami.com/bitnami/ +version: 12.1.5 +options: + auth: + username: superset + password: superset + database: superset diff --git a/stacks/_templates/prometheus-service-monitor.yaml b/stacks/_templates/prometheus-service-monitor.yaml new file mode 100644 index 00000000..15e919c5 --- /dev/null +++ b/stacks/_templates/prometheus-service-monitor.yaml @@ -0,0 +1,36 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: scrape-label + labels: + release: prometheus +spec: + endpoints: + - port: metrics + selector: + matchLabels: + prometheus.io/scrape: "true" + namespaceSelector: + any: true + jobLabel: app.kubernetes.io/instance +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: scrape-minio + labels: + release: prometheus +spec: + endpoints: + - path: /minio/v2/metrics/node + port: http + scheme: http + - path: /minio/v2/metrics/cluster + port: http + scheme: http + selector: + matchLabels: + app: minio + monitoring: "true" + namespaceSelector: + any: true diff --git a/stacks/_templates/prometheus.yaml b/stacks/_templates/prometheus.yaml new file mode 100644 index 00000000..a332afdb --- /dev/null +++ b/stacks/_templates/prometheus.yaml @@ -0,0 +1,55 @@ +releaseName: prometheus +name: kube-prometheus-stack +repo: + name: prometheus-community + url: https://prometheus-community.github.io/helm-charts +version: 41.9.1 +options: + prometheus: + prometheusSpec: + storageSpec: + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 100Gi + service: + type: NodePort + nodePort: null + grafana: + grafana.ini: + analytics: + check_for_updates: false + adminPassword: {{ grafanaAdminPassword }} + persistence: + enabled: true + size: 10Gi + service: + type: NodePort + nodePort: null + dashboardProviders: + stackable-dashboardprovider.yaml: + apiVersion: 1 + providers: + - name: 'stackable' + orgId: 1 + folder: 'Stackable Data Platform' + type: file + disableDeletion: false + allowUiUpdates: true # TODO set to false. Is enabled so Dashboards can be build easier. Note: If a provisioned dashboard is saved from the UI and then later updated from the source, the dashboard stored in the database will always be overwritten + options: + path: /var/lib/grafana/dashboards/stackable # Must be /var/lib/grafana/dashboards/ + dashboardsConfigMaps: + stackable: stackable-grafana-dashboards + kube-state-metrics: + metricLabelsAllowlist: + - pods=[*] + - statefulsets=[*] + - service=[*] + - nodes=[*] + - namespaces=[*] + - namespaces=[*] + - configmaps=[*] + - persistentvolumes=[*] + - persistentvolumeclaims=[*] diff --git a/stacks/_templates/redis-airflow.yaml b/stacks/_templates/redis-airflow.yaml new file mode 100644 index 00000000..fad05f13 --- /dev/null +++ b/stacks/_templates/redis-airflow.yaml @@ -0,0 +1,11 @@ +releaseName: redis-airflow +name: redis +repo: + name: bitnami + url: https://charts.bitnami.com/bitnami/ +version: 17.3.17 +options: + auth: + password: airflow + replica: + replicaCount: 1 diff --git a/stacks/logging/vector-aggregator-discovery.yaml b/stacks/_templates/vector-aggregator-discovery.yaml similarity index 100% rename from stacks/logging/vector-aggregator-discovery.yaml rename to stacks/_templates/vector-aggregator-discovery.yaml diff --git a/stacks/_templates/vector-aggregator.yaml b/stacks/_templates/vector-aggregator.yaml new file mode 100644 index 00000000..6f60a02b --- /dev/null +++ b/stacks/_templates/vector-aggregator.yaml @@ -0,0 +1,51 @@ +releaseName: vector-aggregator +name: vector +repo: + name: vector + url: https://helm.vector.dev +version: 0.18.0 +options: + role: Aggregator + customConfig: + sources: + vector: + address: 0.0.0.0:6000 + type: vector + version: "2" + sinks: + opensearch: + type: elasticsearch + inputs: + - vector + endpoints: + - https://opensearch-cluster-master.default.svc.cluster.local:9200 + mode: bulk + api_version: v8 + tls: + # Add the certificate of the Certificate Authority + # generated by the secret-operator so that the + # OpenSearch service can be verified + ca_file: /certs/ca.crt + auth: + strategy: basic + user: admin + password: {{ openSearchAdminPassword }} + extraVolumeMounts: + # Use the certificate generated by the secret-operator + - name: tls + mountPath: /certs + extraVolumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index 890fb504..089c1553 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -1,3 +1,4 @@ +{% raw %} --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -284,3 +285,4 @@ stringData: connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow connections.celeryResultBackend: db+postgresql://airflow:airflow@postgresql-airflow/airflow connections.celeryBrokerUrl: redis://:airflow@redis-airflow-master:6379/0 +{% endraw %} diff --git a/stacks/monitoring/grafana-dashboards.yaml b/stacks/monitoring/grafana-dashboards.yaml new file mode 100644 index 00000000..73626f13 --- /dev/null +++ b/stacks/monitoring/grafana-dashboards.yaml @@ -0,0 +1,9687 @@ +{% raw %} +apiVersion: v1 +kind: ConfigMap +metadata: + name: stackable-grafana-dashboards +data: + trino.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 31, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU request", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU limit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 35, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory limit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 2, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

$cluster

", + "mode": "html" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cluster name", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 14, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "trino_memory_clustermemorypool_nodes{namespace=~\"$namespace\",job=~\"$cluster\"} - 1 # Subtract 1 for coordinator", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Workers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_cpu_usage_seconds_total{container!=\"\"}[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "request", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(container_memory_working_set_bytes{container!=\"\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_network_receive_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "rx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-sum(max by (namespace, pod)(rate(container_network_transmit_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "tx", + "range": true, + "refId": "B" + } + ], + "title": "Network usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "canceled" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "abandoned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespae, job) (trino_execution_querymanager_runningqueries{namespace=~\"$namespace\",job=~\"$cluster\"})", + "legendFormat": "running", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespae, job) (trino_execution_querymanager_queuedqueries{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "queueud", + "range": true, + "refId": "B" + } + ], + "title": "Running queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "canceled" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "abandoned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespae, job) (trino_execution_querymanager_completedqueries_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "legendFormat": "completed", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespae, job) (trino_execution_querymanager_failedqueries_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "failed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespae, job) (trino_execution_querymanager_abandonedqueries_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "abandoned", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (namespae, job) (trino_execution_querymanager_canceledqueries_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "canceled", + "range": true, + "refId": "C" + } + ], + "title": "Finished queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "trino_memory_clustermemorymanager_clustertotalmemoryreservation{namespace=~\"$namespace\",job=~\"$cluster\"}", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "trino_memory_clustermemorymanager_clustermemorybytes{namespace=~\"$namespace\",job=~\"$cluster\"}", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "C" + } + ], + "title": "Internal memory usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(trino_execution_querymanager_consumedinputrows_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])", + "hide": false, + "legendFormat": "read", + "range": true, + "refId": "A" + } + ], + "title": "Rows/s read", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(trino_execution_querymanager_consumedinputbytes_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])*8", + "hide": false, + "legendFormat": "read", + "range": true, + "refId": "A" + } + ], + "title": "Logical bytes read", + "type": "timeseries" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(trino_memory_clustermemorypool_nodes, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(trino_memory_clustermemorypool_nodes, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "trino", + "value": "trino" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(trino_memory_clustermemorypool_nodes{namespace=~\"$namespace\"}, job)", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(trino_memory_clustermemorypool_nodes{namespace=~\"$namespace\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "hide": 2, + "name": "product", + "query": "trino", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Trino", + "uid": "-mVc4yO4k", + "version": 5, + "weekStart": "" + } + kafka.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU request", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU limit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory limit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 2, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

$cluster

", + "mode": "html" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cluster name", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 14, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(kafka_controller_kafkacontroller_activebrokercount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Brokers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 17, + "y": 0 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(count by (topic) (kafka_server_brokertopicmetrics_messagesin_total{namespace=~\"$namespace\",job=~\"$cluster\",topic!=\"\"}))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Topics", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_cpu_usage_seconds_total{container!=\"\"}[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "request", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(container_memory_working_set_bytes{container!=\"\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_network_receive_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "rx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-sum(max by (namespace, pod)(rate(container_network_transmit_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "tx", + "range": true, + "refId": "B" + } + ], + "title": "Network usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(topic)(rate(kafka_server_brokertopicmetrics_messagesin_total{namespace=~\"$namespace\",job=~\"$cluster\",topic!=\"\"}[$__rate_interval]))", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Messages / topic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(topic)(rate(kafka_server_brokertopicmetrics_bytesin_total{namespace=~\"$namespace\",job=~\"$cluster\",topic!=\"\"}[$__rate_interval]))*8", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes / topic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 18 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(topic)(kafka_log_log_size{namespace=~\"$namespace\",job=~\"$cluster\",topic!=\"\"})", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Log size / topic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 10, + "y": 18 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_used_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_capacity_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "C" + } + ], + "title": "PVC usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 17, + "y": 18 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_server_kafkaserver_linux_disk_read_bytes{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval]))*8", + "hide": false, + "legendFormat": "rx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-sum(rate(kafka_server_kafkaserver_linux_disk_write_bytes{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval]))*8", + "hide": false, + "legendFormat": "tx", + "range": true, + "refId": "B" + } + ], + "title": "Disk IO", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "hide": 2, + "name": "product", + "query": "kafka", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kafka_server_kafkaserver_brokerstate, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kafka_server_kafkaserver_brokerstate, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "kafka", + "value": "kafka" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kafka_server_kafkaserver_brokerstate{namespace=~\"$namespace\"}, job)", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(kafka_server_kafkaserver_brokerstate{namespace=~\"$namespace\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kafka", + "uid": "C_8qIX04k", + "version": 4, + "weekStart": "" + } + minio.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "MinIO Grafana Dashboard - https://min.io/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13502, + "graphTooltip": 0, + "id": 29, + "links": [ + { + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "minio" + ], + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 1, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "time() - max(minio_node_process_starttime_seconds{job=\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 65, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum (minio_s3_traffic_received_bytes{job=\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Traffic Inbound", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 6, + "y": 0 + }, + "id": 50, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "topk(1, max(sum(minio_cluster_capacity_usable_total_bytes{job=\"$scrape_jobs\"}) by (instance))) - topk(1, max(sum(minio_cluster_capacity_usable_free_bytes{job=\"$scrape_jobs\"}) by (instance)))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "A", + "step": 300 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "topk(1, max(sum(minio_cluster_capacity_usable_free_bytes{job=\"$scrape_jobs\"}) by (instance)))", + "hide": false, + "interval": "1m", + "legendFormat": "Free", + "range": true, + "refId": "B" + } + ], + "title": "Capacity", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Objects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Usage" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 0 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(sum(minio_bucket_usage_total_bytes{job=\"$scrape_jobs\"}) by (instance))", + "interval": "", + "legendFormat": "Usage", + "range": true, + "refId": "A" + } + ], + "title": "Data Usage Growth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "semi-dark-red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 16, + "y": 0 + }, + "id": 52, + "links": [], + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "text": {} + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (range) (minio_bucket_objects_size_distribution{job=\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{range}}", + "range": false, + "refId": "A", + "step": 300 + } + ], + "title": "Object size distribution", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 2000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 61, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum (minio_node_file_descriptor_open_total{job=\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Open FDs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 3 + }, + "id": 64, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum (minio_s3_traffic_sent_bytes{job=\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Traffic Outbound", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 2000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 3 + }, + "id": 62, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum without (server,instance,pod) (minio_node_go_routine_total{job=\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Goroutines", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 53, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "minio_cluster_nodes_online_total{job=\"$scrape_jobs\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Online Servers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 9, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(minio_cluster_disk_online_total{job=\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Total online disks in MinIO Cluster", + "metric": "process_start_time_seconds", + "range": false, + "refId": "A", + "step": 60 + } + ], + "title": "Total Online Disks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-yellow", + "value": 75000000 + }, + { + "color": "dark-red", + "value": 100000000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 6 + }, + "id": 66, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "count(count by (bucket) (minio_bucket_usage_total_bytes{job=\"$scrape_jobs\"}))", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Buckets", + "type": "stat" + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bits" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 6 + }, + "hiddenSeries": false, + "id": 63, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))*8", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Received [{{server}}]", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Data Received Rate ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "bits", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bits" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 6 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))*8", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Sent [{{server}}]", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Data Sent Rate ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "bits", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 8 + }, + "id": 69, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "minio_cluster_nodes_offline_total{job=\"$scrape_jobs\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Offline Servers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 8 + }, + "id": 78, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_disk_offline_total{job=\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Offline Disks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-yellow", + "value": 75000000 + }, + { + "color": "dark-red", + "value": 100000000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 9 + }, + "id": 44, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "topk(1, max(sum(minio_bucket_usage_object_total{job=\"$scrape_jobs\"}) by (instance)))", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Objects", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 10 + }, + "id": 80, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(minio_heal_time_last_activity_nano_seconds{job=\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Heal Activity", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 10 + }, + "id": 81, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(minio_usage_last_activity_nano_seconds{job=\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Scan Activity", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total s3 bytes sent per bucket", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 90, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(bucket) (rate(minio_bucket_traffic_sent_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))*8", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bucket Traffic Sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total s3 bytes received per bucket", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 92, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(bucket) (rate(minio_bucket_traffic_received_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))*8", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bucket Traffic Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total s3 bytes sent per bucket", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (bucket) (minio_bucket_usage_object_total{job=\"$scrape_jobs\"})", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Bucket objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total s3 bytes sent per bucket", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "lakehouse" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 94, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (bucket) (minio_bucket_usage_total_bytes{job=\"$scrape_jobs\"})", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Bucket size", + "type": "timeseries" + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 86, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Error Rate (5xx)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 42 + }, + "hiddenSeries": false, + "id": 88, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API Request Error Rate (4xx)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "10.13.1.25:9000 DELETE": "red", + "10.13.1.25:9000 GET": "green", + "10.13.1.25:9000 POST": "blue" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of bytes received and sent among all MinIO server instances", + "fieldConfig": { + "defaults": { + "links": [], + "unit": "binbps" + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_sent_bytes{job=\"$scrape_jobs\"}[$__rate_interval])*8", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Internode Bytes Received [{{server}}]", + "metric": "minio_http_requests_duration_seconds_count", + "range": true, + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_received_bytes{job=\"$scrape_jobs\"}[$__rate_interval])*8", + "interval": "", + "legendFormat": "Internode Bytes Sent [{{server}}]", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Internode Data Transfer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:211", + "format": "binbps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:212", + "format": "s", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 52 + }, + "hiddenSeries": false, + "id": 84, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (instance) (minio_heal_objects_heal_total{job=\"$scrape_jobs\"})", + "interval": "", + "legendFormat": "Objects healed in current self heal run", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (instance) (minio_heal_objects_error_total{job=\"$scrape_jobs\"})", + "hide": false, + "interval": "", + "legendFormat": "Heal errors in current self heal run", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (instance) (minio_heal_objects_total{job=\"$scrape_jobs\"}) ", + "hide": false, + "interval": "", + "legendFormat": "Objects scanned in current self heal run", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Healing", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:846", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:847", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 61 + }, + "hiddenSeries": false, + "id": 77, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(minio_node_process_cpu_total_seconds{job=\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "CPU Usage Rate [{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Node CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1043", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1044", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 61 + }, + "hiddenSeries": false, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "minio_node_process_resident_memory_bytes{job=\"$scrape_jobs\"}", + "interval": "", + "legendFormat": "Memory Used [{{server}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Node Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1043", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1044", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 74, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "minio_node_disk_used_bytes{job=\"$scrape_jobs\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Used Capacity [{{server}}:{{disk}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Drive Used Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:381", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:382", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "hiddenSeries": false, + "id": 82, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "minio_cluster_disk_free_inodes{job=\"$scrape_jobs\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Free Inodes [{{server}}:{{disk}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Drives Free Inodes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:381", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:382", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Offline 10.13.1.25:9000": "dark-red", + "Total 10.13.1.25:9000": "blue" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of online disks per MinIO Server", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(minio_node_syscall_read_total{job=\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Read Syscalls [{{server}}]", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(minio_node_syscall_write_total{job=\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Write Syscalls [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Node Syscalls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:185", + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:186", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "available 10.13.1.25:9000": "green", + "used 10.13.1.25:9000": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 78 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "minio_node_file_descriptor_open_total{job=\"$scrape_jobs\"}", + "interval": "", + "legendFormat": "Open FDs [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Node File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:213", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "binbps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 87 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_node_io_rchar_bytes{job=\"$scrape_jobs\"}[$__rate_interval])*8", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Node RChar [{{server}}]", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_node_io_wchar_bytes{job=\"$scrape_jobs\"}[$__rate_interval])*8", + "interval": "", + "legendFormat": "Node WChar [{{server}}]", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Node IO", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:381", + "format": "binbps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:382", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [ + "minio" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "minio", + "value": "minio" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(minio_cluster_disk_total,job)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "scrape_jobs", + "options": [], + "query": { + "query": "label_values(minio_cluster_disk_total,job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "MinIO", + "uid": "TgmJnqnnk", + "version": 2, + "weekStart": "" + } + hdfs.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU request", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU limit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory limit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 2, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

$cluster

", + "mode": "html" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cluster name", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 14, + "y": 0 + }, + "id": 25, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(hadoop_journalnode_version{namespace=~\"$namespace\",job=~\"$cluster\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Journalnodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 17, + "y": 0 + }, + "id": 26, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(hadoop_namenode_version{namespace=~\"$namespace\",job=~\"$cluster\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Namenodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 20, + "y": 0 + }, + "id": 27, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(hadoop_datanode_version{namespace=~\"$namespace\",job=~\"$cluster\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Datanodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_cpu_usage_seconds_total{container!=\"\"}[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "request", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(container_memory_working_set_bytes{container!=\"\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_network_receive_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "rx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-sum(max by (namespace, pod)(rate(container_network_transmit_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "tx", + "range": true, + "refId": "B" + } + ], + "title": "Network usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-green", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "remaining" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 0, + "y": 10 + }, + "id": 29, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(hadoop_namenode_capacity_remaining{namespace=~\"$namespace\",job=~\"$cluster\"})", + "legendFormat": "remaining", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(hadoop_namenode_capacity_used{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "B" + } + ], + "title": "DFS usage", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 4, + "y": 10 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(hadoop_namenode_capacity_total{namespace=~\"$namespace\",job=~\"$cluster\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DFS capacity", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 7, + "y": 10 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by(pod)(rate(hadoop_datanode_bytes_read{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))*8", + "hide": false, + "legendFormat": "rx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-sum(max by(pod)(rate(hadoop_datanode_bytes_written{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))*8", + "hide": false, + "legendFormat": "tx", + "range": true, + "refId": "B" + } + ], + "title": "HDFS cluster I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 10, + "x": 14, + "y": 10 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(hadoop_namenode_files_total{namespace=~\"$namespace\",job=~\"$cluster\"})", + "legendFormat": "total", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(hadoop_namenode_num_files_under_construction{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "under construction", + "range": true, + "refId": "B" + } + ], + "title": "Files", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 4, + "y": 13 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(hadoop_namenode_capacity_used{namespace=~\"$namespace\",job=~\"$cluster\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DFS used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "corrupt" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "mssing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "underreplicated" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "excess" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 10, + "x": 14, + "y": 15 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(hadoop_namenode_blocks_total{namespace=~\"$namespace\",job=~\"$cluster\"})", + "legendFormat": "total", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(hadoop_namenode_missing_blocks{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "mssing", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(hadoop_namenode_corrupt_blocks{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "corrupt", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(hadoop_namenode_under_replicated_blocks{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "underreplicated", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(hadoop_namenode_excess_blocks{namespace=~\"$namespace\",job=~\"$cluster\"})", + "hide": false, + "legendFormat": "excess", + "range": true, + "refId": "E" + } + ], + "title": "Blocks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 4, + "y": 16 + }, + "id": 31, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(hadoop_namenode_capacity_remaining{namespace=~\"$namespace\",job=~\"$cluster\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DFS remaining", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(hadoop_namenode_total_load{namespace=~\"$namespace\",job=~\"$cluster\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Client connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 19 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_used_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_capacity_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "C" + } + ], + "title": "PVC usage", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "hide": 2, + "name": "product", + "query": "hdfs", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(hadoop_namenode_blocks_total, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(hadoop_namenode_blocks_total, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "hdfs", + "value": "hdfs" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(hadoop_namenode_blocks_total{namespace=~\"$namespace\"}, job)", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(hadoop_namenode_blocks_total{namespace=~\"$namespace\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "HDFS", + "uid": "qjZGZ3AVz", + "version": 4, + "weekStart": "" + } + hbase.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 32, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU request", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU limit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory limit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 2, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

$cluster

", + "mode": "html" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cluster name", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 14, + "y": 0 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(Hadoop_HBase_numMasterWALs{namespace=~\"$namespace\",job=~\"$cluster\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Masters", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 17, + "y": 0 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(Hadoop_HBase_regionCount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Region servers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_cpu_usage_seconds_total{container!=\"\"}[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "request", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(container_memory_working_set_bytes{container!=\"\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_network_receive_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "rx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-sum(max by (namespace, pod)(rate(container_network_transmit_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "tx", + "range": true, + "refId": "B" + } + ], + "title": "Network usage", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "hide": 2, + "name": "product", + "query": "hbase", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(Hadoop_HBase_numTables, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(Hadoop_HBase_numTables, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "hbase", + "value": "hbase" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(Hadoop_HBase_numTables{namespace=~\"$namespace\"}, job)", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(Hadoop_HBase_numTables{namespace=~\"$namespace\"}, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "HBase", + "uid": "vVpFeeAVz", + "version": 6, + "weekStart": "" + } +{% endraw %} diff --git a/stacks/stacks-v1.yaml b/stacks/stacks-v1.yaml index f3ae302d..b346c1f7 100644 --- a/stacks/stacks-v1.yaml +++ b/stacks/stacks-v1.yaml @@ -1,706 +1,6 @@ ---- -_templates: - - helmChart: &template-minio-druid - releaseName: minio-druid - name: minio - repo: - name: minio - url: https://charts.min.io/ - version: 4.0.15 - options: - rootUser: admin - rootPassword: adminadmin - mode: standalone - persistence: - size: 15Gi - users: - - accessKey: druid - secretKey: druiddruid - policy: readwrite - buckets: - - name: druid - policy: public - resources: - requests: - memory: 2Gi - service: - type: NodePort - nodePort: null - consoleService: - type: NodePort - nodePort: null - - helmChart: &template-minio-trino - releaseName: minio-trino - name: minio - repo: - name: minio - url: https://charts.min.io/ - version: 4.0.15 - options: - rootUser: admin - rootPassword: adminadmin - mode: standalone - persistence: - size: 10Gi - users: - - accessKey: trino - secretKey: trinotrino - policy: readwrite - - accessKey: hive - secretKey: hivehive - policy: readwrite - - accessKey: demo - secretKey: demodemo - policy: readwrite - buckets: - - name: demo - policy: public - resources: - requests: - cpu: 500m - memory: 1Gi - service: - type: NodePort - nodePort: null - consoleService: - type: NodePort - nodePort: null - - helmChart: &template-minio-distributed - releaseName: minio - name: minio - repo: - name: minio - url: https://charts.min.io/ - version: 4.0.15 - options: - rootUser: admin - rootPassword: adminadmin - mode: distributed - replicas: 5 - persistence: - size: 250Gi - users: - - accessKey: trino - secretKey: trinotrino - policy: readwrite - - accessKey: hive - secretKey: hivehive - policy: readwrite - - accessKey: demo - secretKey: demodemo - policy: readwrite - buckets: - - name: staging - policy: public - - name: lakehouse - policy: public - resources: - requests: - cpu: 500m - memory: 2Gi - service: - type: NodePort - nodePort: null - consoleService: - type: NodePort - nodePort: null - - helmChart: &template-minio-distributed-small - releaseName: minio - name: minio - repo: - name: minio - url: https://charts.min.io/ - version: 4.0.15 - options: - rootUser: admin - rootPassword: adminadmin - mode: distributed - replicas: 2 - persistence: - size: 50Gi - users: - - accessKey: minio - secretKey: miniominio - policy: readwrite - buckets: - - name: lakehouse - policy: public - resources: - requests: - cpu: 500m - memory: 2Gi - service: - type: NodePort - nodePort: null - consoleService: - type: NodePort - nodePort: null - - helmChart: &template-postgresql-hive - releaseName: postgresql-hive - name: postgresql - repo: - name: bitnami - url: https://charts.bitnami.com/bitnami/ - version: 12.1.5 - options: - auth: - username: hive - password: hive - database: hive - primary: - extendedConfiguration: password_encryption=md5 - - helmChart: &template-postgresql-hive-iceberg - releaseName: postgresql-hive-iceberg - name: postgresql - repo: - name: bitnami - url: https://charts.bitnami.com/bitnami/ - version: 12.1.5 - options: - auth: - username: hive - password: hive - database: hive - primary: - extendedConfiguration: password_encryption=md5 - - helmChart: &template-postgresql-hivehdfs - releaseName: hivehdfs - name: postgresql - repo: - name: bitnami - url: https://charts.bitnami.com/bitnami/ - version: 12.1.5 - options: - auth: - username: hive - password: hive - database: hivehdfs - primary: - extendedConfiguration: password_encryption=md5 - - helmChart: &template-postgresql-hives3 - releaseName: hives3 - name: postgresql - repo: - name: bitnami - url: https://charts.bitnami.com/bitnami/ - version: 12.1.5 - options: - auth: - username: hive - password: hive - database: hives3 - primary: - extendedConfiguration: password_encryption=md5 - - helmChart: &template-postgresql-druid - releaseName: postgresql-druid - name: postgresql - repo: - name: bitnami - url: https://charts.bitnami.com/bitnami/ - version: 12.1.5 - options: - auth: - username: druid - password: druid - database: druid - - helmChart: &template-postgresql-superset - releaseName: postgresql-superset - name: postgresql - repo: - name: bitnami - url: https://charts.bitnami.com/bitnami/ - version: 12.1.5 - options: - auth: - username: superset - password: superset - database: superset - - helmChart: &template-postgresql-airflow - releaseName: postgresql-airflow - name: postgresql - repo: - name: bitnami - url: https://charts.bitnami.com/bitnami/ - version: 12.1.5 - options: - auth: - username: airflow - password: airflow - database: airflow - - helmChart: &template-redis-airflow - releaseName: redis-airflow - name: redis - repo: - name: bitnami - url: https://charts.bitnami.com/bitnami/ - version: 17.3.17 - options: - auth: - password: airflow - replica: - replicaCount: 1 - - helmChart: &template-opensearch - releaseName: opensearch - name: opensearch - repo: - name: opensearch - url: https://opensearch-project.github.io/helm-charts - version: 2.9.0 - options: - config: - opensearch.yml: | - plugins: - security: - # Use default security settings - allow_default_init_securityindex: true - # Allow communication between the nodes which use the - # certificates generated by the secret-operator - nodes_dn: - - CN=generated certificate for pod - # Use the certificate generated by the secret-operator - ssl: - http: - # Enable TLS on the REST layer - enabled: true - pemcert_filepath: certs/tls.crt - pemkey_filepath: certs/tls.key - pemtrustedcas_filepath: certs/ca.crt - transport: - pemcert_filepath: certs/tls.crt - pemkey_filepath: certs/tls.key - pemtrustedcas_filepath: certs/ca.crt - # Disable the verification of hostnames because - # internal IPs are used which are not included in - # the certificates generated by the secret-operator. - enforce_hostname_verification: false - securityConfig: - path: /usr/share/opensearch/config/opensearch-security - internalUsersSecret: opensearch-internal-users - extraEnvs: - # Disable the creation of demo certificates - - name: DISABLE_INSTALL_DEMO_CONFIG - value: "true" - extraVolumeMounts: - # Mount the certificate generated by the secret-operator - - name: tls - mountPath: /usr/share/opensearch/config/certs - extraVolumes: - # Request a TLS certificate from the secret-operator - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - # Add the service opensearch-cluster-master to the - # distinguished names because this service is used - # by Vector. - secrets.stackable.tech/scope: |- - service=opensearch-cluster-master - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - extraObjects: - - apiVersion: v1 - kind: Secret - metadata: - name: opensearch-internal-users - stringData: - internal_users.yml: | - --- - # This is the internal user database. - # The hash value is a bcrypt hash. - - _meta: - type: "internalusers" - config_version: 2 - - admin: - hash: "$2y$12$H8Sro/AECPTvvM8iUYKIEO74AhHm46McG3BGntSkNlVkdjB7cQUo6" - reserved: true - backend_roles: - - "admin" - description: "Demo admin user" - - kibanaserver: - hash: "$2y$12$M9Yw0bLcDjadK0gd6r00Ye5PQztHP9ZzxkRo4AVCvxABFU/mVWGgu" - reserved: true - description: "Demo OpenSearch Dashboards user" - - apiVersion: v1 - kind: Secret - metadata: - name: opensearch-user - stringData: - username: admin - password: adminadmin - - apiVersion: v1 - kind: Secret - metadata: - name: opensearch-dashboards-user - stringData: - username: kibanaserver - password: kibanaserverkibanaserver - cookie: NmOcOdfZbEMEHPwri7Y6Gybhw2k3O9UX - - helmChart: &template-opensearch-dashboards - releaseName: opensearch-dashboards - name: opensearch-dashboards - repo: - name: opensearch-dashboards - url: https://opensearch-project.github.io/helm-charts - version: 2.7.0 - options: - service: - type: NodePort - port: 5601 - annotations: - stackable.tech/logging-view-logs: |- - /app/discover?security_tenant=global#/view/logs - stackable.tech/logging-credentials-secret: opensearch-user - opensearchAccount: - secret: opensearch-dashboards-user - lifecycle: - postStart: - exec: - command: - - bash - - -c - - | - #!/bin/bash - curl \ - --retry-connrefused \ - --retry 120 \ - --retry-delay 5 \ - --header "Content-Type:application/json" \ - --header "osd-xsrf:true" \ - --header "securitytenant: global" \ - --data '{ - "attributes": { - "title": "vector-*", - "timeFieldName": "timestamp" - } - }' \ - "http://admin:adminadmin@127.0.0.1:5601/api/saved_objects/index-pattern/vector-*" - curl \ - --header "Content-Type:application/json" \ - --header "osd-xsrf:true" \ - --header "securitytenant: global" \ - --data '{ - "attributes": { - "title":"Logs", - "columns": ["pod", "container", "logger", "level", "message"], - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"indexRefName\": \"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "id": "vector-*", - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern" - } - ] - }' \ - http://admin:adminadmin@127.0.0.1:5601/api/saved_objects/search/logs - - helmChart: &template-vector-aggregator - releaseName: vector-aggregator - name: vector - repo: - name: vector - url: https://helm.vector.dev - version: 0.18.0 - options: - role: Aggregator - customConfig: - sources: - vector: - address: 0.0.0.0:6000 - type: vector - version: "2" - sinks: - opensearch: - type: elasticsearch - inputs: - - vector - endpoints: - - https://opensearch-cluster-master.default.svc.cluster.local:9200 - mode: bulk - api_version: v8 - tls: - # Add the certificate of the Certificate Authority - # generated by the secret-operator so that the - # OpenSearch service can be verified - ca_file: /certs/ca.crt - auth: - strategy: basic - user: admin - password: adminadmin - extraVolumeMounts: - # Use the certificate generated by the secret-operator - - name: tls - mountPath: /certs - extraVolumes: - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: pod - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - - - helmChart: &template-jupyterhub - releaseName: jupyterhub - name: jupyterhub - repo: - name: jupyterhub - url: https://jupyterhub.github.io/helm-chart/ - version: 2.0.0 - options: - proxy: - service: - type: NodePort - hub: - config: - Authenticator: - allowed_users: - - admin - DummyAuthenticator: - password: adminadmin - JupyterHub: - authenticator_class: dummy - rbac: - create: true - singleuser: - serviceAccountName: spark - extraEnv: - HADOOP_CONF_DIR: "/home/jovyan/hdfs" - initContainers: - - name: download-notebook - image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.4.0 - command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb -o /notebook/notebook.ipynb'] - volumeMounts: - - mountPath: /notebook - name: notebook - storage: - extraVolumes: - - name: hdfs-discovery-configmap - configMap: - name: hdfs - - name: notebook - emptyDir: - sizeLimit: 500Mi - extraVolumeMounts: - - name: hdfs-discovery-configmap - mountPath: /home/jovyan/hdfs - - name: notebook - mountPath: /home/jovyan/notebook - image: - # You should replace the "latest" tag with a fixed version from: - # https://hub.docker.com/r/jupyter/datascience-notebook/tags/ - # Inspect the Dockerfile at: - # https://github.com/jupyter/docker-stacks/tree/HEAD/datascience-notebook/Dockerfile - name: jupyter/pyspark-notebook - tag: python-3.9 - # `cmd: null` allows the custom CMD of the Jupyter docker-stacks to be used - # which performs further customization on startup. - cmd: null stacks: - airflow: - description: Stack containing Airflow scheduling platform - stackableRelease: 23.1 - labels: - - airflow - manifests: - - helmChart: *template-postgresql-airflow - - helmChart: *template-redis-airflow - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/airflow/airflow.yaml - data-lakehouse-iceberg-trino-spark: - description: Data lakehouse using Iceberg lakehouse on S3, Trino as query engine, Spark for streaming ingest and Superset for data visualization - stackableRelease: 23.1 - labels: - - iceberg - - trino - - spark - - superset - - kafka - - nifi - - minio - - s3 - manifests: - - helmChart: *template-minio-distributed - - helmChart: *template-postgresql-hive - - helmChart: *template-postgresql-hive-iceberg - - helmChart: *template-postgresql-superset - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/data-lakehouse-iceberg-trino-spark/zookeeper.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/kafka-druid-superset-s3/superset.yaml # Reuse - hdfs-hbase: - description: HBase cluster using HDFS as underlying storage - stackableRelease: 23.1 - labels: - - hbase - - hdfs - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/hdfs-hbase/zookeeper.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/hdfs-hbase/hdfs.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/hdfs-hbase/hbase.yaml - kafka-druid-superset-s3: - description: Stack containing Kafka, MinIO, Druid and Superset for data visualization - stackableRelease: 23.1 - labels: - - kafka - - druid - - superset - - minio - - s3 - manifests: - - helmChart: *template-minio-druid - - helmChart: *template-postgresql-druid - - helmChart: *template-postgresql-superset - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/kafka-druid-superset-s3/zookeeper.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/kafka-druid-superset-s3/kafka.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/kafka-druid-superset-s3/druid.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/kafka-druid-superset-s3/superset.yaml - nifi-kafka-druid-superset-s3: - description: Stack containing NiFi, Kafka, Druid, MinIO and Superset for data visualization - stackableRelease: 23.1 - labels: - - nifi - - kafka - - druid - - superset - - minio - - s3 - manifests: - - helmChart: *template-minio-druid - - helmChart: *template-postgresql-druid - - helmChart: *template-postgresql-superset - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/kafka-druid-superset-s3/zookeeper.yaml # Reuse - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/kafka-druid-superset-s3/kafka.yaml # Reuse - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/kafka-druid-superset-s3/druid.yaml # Reuse - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/kafka-druid-superset-s3/superset.yaml # Reuse - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/nifi-kafka-druid-superset-s3/nifi.yaml - spark-trino-superset-s3: - description: Stack containing MinIO, Trino and Superset for data visualization - stackableRelease: 23.1 - labels: - - trino - - superset - - minio - - s3 - manifests: - - helmChart: *template-minio-trino - - helmChart: *template-postgresql-hive - - helmChart: *template-postgresql-hive-iceberg - - helmChart: *template-postgresql-superset - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/spark-trino-superset-s3/hive-metastore.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/spark-trino-superset-s3/trino.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/spark-trino-superset-s3/superset.yaml - trino-superset-s3: - description: Stack containing MinIO, Trino and Superset for data visualization - stackableRelease: 23.1 - labels: - - trino - - superset - - minio - - s3 - manifests: - - helmChart: *template-minio-trino - - helmChart: *template-postgresql-hive - - helmChart: *template-postgresql-superset - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/trino-superset-s3/hive-metastore.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/trino-superset-s3/trino.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/trino-superset-s3/superset.yaml - trino-iceberg: - description: Stack containing Trino using Apache Iceberg as a S3 data lakehouse - stackableRelease: 23.1 - labels: - - trino - - iceberg - - minio - - s3 - manifests: - - helmChart: *template-minio-distributed-small - - helmChart: *template-postgresql-hive-iceberg - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/trino-iceberg/s3-connection.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/trino-iceberg/hive-metastores.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/trino-iceberg/trino.yaml - jupyterhub-pyspark-hdfs: - description: Jupyterhub with PySpark and HDFS integration - stackableRelease: 23.1 - labels: - - jupyterhub - - hdfs - - pyspark - manifests: - - helmChart: *template-jupyterhub - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/jupyterhub-pyspark-hdfs/zookeeper.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/jupyterhub-pyspark-hdfs/hdfs.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/jupyterhub-pyspark-hdfs/serviceaccount.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/jupyterhub-pyspark-hdfs/spark_driver_service.yaml - dual-hive-hdfs-s3: - description: Dual stack Hive on HDFS and S3 for Hadoop/Hive to Trino migration - stackableRelease: 23.1 - labels: - - trino - - hive - - hdfs - - s3 - manifests: - - helmChart: *template-postgresql-hivehdfs - - helmChart: *template-postgresql-hives3 - - helmChart: *template-minio-trino - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/dual-hive-hdfs-s3/hdfs.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/dual-hive-hdfs-s3/hive.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/dual-hive-hdfs-s3/trino.yaml - logging: - description: Stack containing OpenSearch, OpenSearch Dashboards, and Vector - stackableRelease: 23.1 - labels: - - logging - - opensearch - - opensearch-dashboards - - vector - manifests: - - helmChart: *template-opensearch - - helmChart: *template-opensearch-dashboards - - helmChart: *template-vector-aggregator - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/logging/vector-aggregator-discovery.yaml - tutorial-openldap: - description: >- - An OpenLDAP instance with two users (alice:alice, bob:bob) and TLS enabled. - The bind user credentials are: ldapadmin:ldapadminpassword. - No AuthenticationClass is configured, The AuthenticationClass is created manually in the tutorial. - Use the 'openldap' Stack for an OpenLDAD with an AuthenticationClass already installed. - stackableRelease: 23.1 - labels: - - authentication - - ldap - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/authentication/openldap-tls.yaml - openldap: - description: >- - An OpenLDAP instance with two users (alice:alice, bob:bob) and TLS enabled. - The bind user credentials are: ldapadmin:ldapadminpassword. - The LDAP AuthenticationClass is called 'ldap' and the SecretClass for the bind credentials is called 'ldap-bind-credentials'. - The stack already creates an appropriate Secret, so referring to the 'ldap' AuthenticationClass in your ProductCluster should be enough. - stackableRelease: 23.1 - labels: - - authentication - - ldap - manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/authentication/openldap-tls.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/stacks/authentication/openldap-tls-authenticationclass.yaml + please-update: + description: This version of stackablectl is outdated, please visit https://docs.stackable.tech/stackablectl/stable/installation.html on how to get the latest version + stackableRelease: does-not-exist + labels: [] + manifests: [] diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml new file mode 100644 index 00000000..23c2ef35 --- /dev/null +++ b/stacks/stacks-v2.yaml @@ -0,0 +1,318 @@ +stacks: + monitoring: + description: Stack containing Prometheus and Grafana + stackableRelease: 23.1 + stackableOperators: + - commons + labels: + - monitoring + - prometheus + - grafana + manifests: + - plainYaml: stacks/monitoring/grafana-dashboards.yaml + - helmChart: stacks/_templates/prometheus.yaml + - plainYaml: stacks/_templates/prometheus-service-monitor.yaml + parameters: + - name: grafanaAdminPassword + description: Password of the Grafana admin user + default: adminadmin + logging: + description: Stack containing OpenSearch, OpenSearch Dashboards (Kibana) and Vector aggregator + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - zookeeper # demo does install a zookeeper to produce logs + labels: + - logging + - opensearch + - opensearch-dashboards + - vector + manifests: + - helmChart: stacks/_templates/opensearch.yaml + - helmChart: stacks/_templates/opensearch-dashboards.yaml + - helmChart: stacks/_templates/vector-aggregator.yaml + - plainYaml: stacks/_templates/vector-aggregator-discovery.yaml + parameters: + - name: openSearchAdminPassword + description: Password of the OpenSearch admin user + default: adminadmin + - name: openSearchDashboardPassword + description: Password of OpenSearch Dashboard user + default: kibanaserverkibanaserver + - name: setSysctlMaxMapCount + description: Wether an init-container should be used to increase 'sysctl -w vm.max_map_count'. This requires to spawn an init-container with runsAsUser 0 privileges, which some clusters prohibit. You can increase 'vm.max_map_count' on all of the kubernetes nodes manually and set this to 'false'. + default: "true" + airflow: + description: Stack containing Airflow scheduling platform + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - airflow + - spark-k8s # Some demo does schedule a Spark job + labels: + - airflow + manifests: + - helmChart: stacks/_templates/postgresql-airflow.yaml + - helmChart: stacks/_templates/redis-airflow.yaml + - plainYaml: stacks/airflow/airflow.yaml + # TODO: Parameterize + parameters: [] + data-lakehouse-iceberg-trino-spark: + description: Data lakehouse using Iceberg lakehouse on S3, Trino as query engine, Spark for streaming ingest and Superset for data visualization + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - hive + - trino + - zookeeper + - kafka + - nifi + - superset + - spark-k8s + labels: + - iceberg + - trino + - spark + - superset + - kafka + - nifi + - minio + - s3 + manifests: + - helmChart: stacks/_templates/minio-distributed.yaml + - helmChart: stacks/_templates/postgresql-hive.yaml + - helmChart: stacks/_templates/postgresql-hive-iceberg.yaml + - helmChart: stacks/_templates/postgresql-superset.yaml + - plainYaml: stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml + - plainYaml: stacks/data-lakehouse-iceberg-trino-spark/trino.yaml + - plainYaml: stacks/data-lakehouse-iceberg-trino-spark/zookeeper.yaml + - plainYaml: stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml + - plainYaml: stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml + - plainYaml: stacks/kafka-druid-superset-s3/superset.yaml # Reuse + # TODO: Parameterize + parameters: [] + hdfs-hbase: + description: HBase cluster using HDFS as underlying storage + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - zookeeper + - hdfs + - hbase + labels: + - hbase + - hdfs + manifests: + - plainYaml: stacks/hdfs-hbase/zookeeper.yaml + - plainYaml: stacks/hdfs-hbase/hdfs.yaml + - plainYaml: stacks/hdfs-hbase/hbase.yaml + # TODO: Parameterize + parameters: [] + kafka-druid-superset-s3: + description: Stack containing Kafka, MinIO, Druid and Superset for data visualization + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - zookeeper + - kafka + - druid + - superset + labels: + - kafka + - druid + - superset + - minio + - s3 + manifests: + - helmChart: stacks/_templates/minio-druid.yaml + - helmChart: stacks/_templates/postgresql-druid.yaml + - helmChart: stacks/_templates/postgresql-superset.yaml + - plainYaml: stacks/kafka-druid-superset-s3/zookeeper.yaml + - plainYaml: stacks/kafka-druid-superset-s3/kafka.yaml + - plainYaml: stacks/kafka-druid-superset-s3/druid.yaml + - plainYaml: stacks/kafka-druid-superset-s3/superset.yaml + # TODO: Parameterize + parameters: [] + nifi-kafka-druid-superset-s3: + description: Stack containing NiFi, Kafka, Druid, MinIO and Superset for data visualization + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - zookeeper + - kafka + - druid + - superset + - nifi + labels: + - nifi + - kafka + - druid + - superset + - minio + - s3 + manifests: + - helmChart: stacks/_templates/minio-druid.yaml + - helmChart: stacks/_templates/postgresql-druid.yaml + - helmChart: stacks/_templates/postgresql-superset.yaml + - plainYaml: stacks/kafka-druid-superset-s3/zookeeper.yaml # Reuse + - plainYaml: stacks/kafka-druid-superset-s3/kafka.yaml # Reuse + - plainYaml: stacks/kafka-druid-superset-s3/druid.yaml # Reuse + - plainYaml: stacks/kafka-druid-superset-s3/superset.yaml # Reuse + - plainYaml: stacks/nifi-kafka-druid-superset-s3/nifi.yaml + # TODO: Parameterize + parameters: [] + spark-trino-superset-s3: + description: Stack containing MinIO, Trino and Superset for data visualization + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - hive + - trino + - superset + - spark-k8s # Some demo does schedule a Spark job + labels: + - trino + - superset + - minio + - s3 + manifests: + - helmChart: stacks/_templates/minio-trino.yaml + - helmChart: stacks/_templates/postgresql-hive.yaml + - helmChart: stacks/_templates/postgresql-hive-iceberg.yaml + - helmChart: stacks/_templates/postgresql-superset.yaml + - plainYaml: stacks/spark-trino-superset-s3/hive-metastore.yaml + - plainYaml: stacks/spark-trino-superset-s3/trino.yaml + - plainYaml: stacks/spark-trino-superset-s3/superset.yaml + # TODO: Parameterize + parameters: [] + trino-superset-s3: + description: Stack containing MinIO, Trino and Superset for data visualization + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - hive + - trino + - superset + labels: + - trino + - superset + - minio + - s3 + manifests: + - helmChart: stacks/_templates/minio-trino.yaml + - helmChart: stacks/_templates/postgresql-hive.yaml + - helmChart: stacks/_templates/postgresql-superset.yaml + - plainYaml: stacks/trino-superset-s3/hive-metastore.yaml + - plainYaml: stacks/trino-superset-s3/trino.yaml + - plainYaml: stacks/trino-superset-s3/superset.yaml + # TODO: Parameterize + parameters: [] + trino-iceberg: + description: Stack containing Trino using Apache Iceberg as a S3 data lakehouse + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - hive + - trino + labels: + - trino + - iceberg + - minio + - s3 + manifests: + - helmChart: stacks/_templates/minio-distributed-small.yaml + - helmChart: stacks/_templates/postgresql-hive-iceberg.yaml + - plainYaml: stacks/trino-iceberg/s3-connection.yaml + - plainYaml: stacks/trino-iceberg/hive-metastores.yaml + - plainYaml: stacks/trino-iceberg/trino.yaml + # TODO: Parameterize + parameters: [] + jupyterhub-pyspark-hdfs: + description: Jupyterhub with PySpark and HDFS integration + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - zookeeper + - hdfs + - spark-k8s + labels: + - jupyterhub + - hdfs + - pyspark + manifests: + - helmChart: stacks/_templates/jupyterhub.yaml + - plainYaml: stacks/jupyterhub-pyspark-hdfs/zookeeper.yaml + - plainYaml: stacks/jupyterhub-pyspark-hdfs/hdfs.yaml + - plainYaml: stacks/jupyterhub-pyspark-hdfs/serviceaccount.yaml + - plainYaml: stacks/jupyterhub-pyspark-hdfs/spark_driver_service.yaml + # TODO: Parameterize + parameters: [] + dual-hive-hdfs-s3: + description: Dual stack Hive on HDFS and S3 for Hadoop/Hive to Trino migration + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + - zookeeper + - hdfs + - hive + - trino + labels: + - trino + - hive + - hdfs + - s3 + manifests: + - helmChart: stacks/_templates/postgresql-hivehdfs.yaml + - helmChart: stacks/_templates/postgresql-hives3.yaml + - helmChart: stacks/_templates/minio-trino.yaml + - plainYaml: stacks/dual-hive-hdfs-s3/hdfs.yaml + - plainYaml: stacks/dual-hive-hdfs-s3/hive.yaml + - plainYaml: stacks/dual-hive-hdfs-s3/trino.yaml + # TODO: Parameterize + parameters: [] + tutorial-openldap: + description: >- + An OpenLDAP instance with two users (alice:alice, bob:bob) and TLS enabled. + The bind user credentials are: ldapadmin:ldapadminpassword. + No AuthenticationClass is configured, The AuthenticationClass is created manually in the tutorial. + Use the 'openldap' Stack for an OpenLDAD with an AuthenticationClass already installed. + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + labels: + - authentication + - ldap + manifests: + - plainYaml: stacks/authentication/openldap-tls.yaml + # TODO: Parameterize + parameters: [] + openldap: + description: >- + An OpenLDAP instance with two users (alice:alice, bob:bob) and TLS enabled. + The bind user credentials are: ldapadmin:ldapadminpassword. + The LDAP AuthenticationClass is called 'ldap' and the SecretClass for the bind credentials is called 'ldap-bind-credentials'. + The stack already creates an appropriate Secret, so referring to the 'ldap' AuthenticationClass in your ProductCluster should be enough. + stackableRelease: 23.1 + stackableOperators: + - commons + - secret + labels: + - authentication + - ldap + manifests: + - plainYaml: stacks/authentication/openldap-tls.yaml + - plainYaml: stacks/authentication/openldap-tls-authenticationclass.yaml + # TODO: Parameterize + parameters: []