[Merged by Bors] - Logging

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Added
+
+- Support listing OpenSearch Dashboards services ([#187](https://github.com/stackabletech/stackablectl/pull/187)).
+
 ### Changed
 
 - Print CLI tables using [comfy-table](https://crates.io/crates/comfy-table) ([#176](https://github.com/stackabletech/stackablectl/pull/176))

demos/demos-v1.yaml

@@ -105,3 +105,15 @@ demos:
       - plainYaml: https://github.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml
       - plainYaml: https://github.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
       - plainYaml: https://github.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml
+  logging:
+    description: Demo showing the logging stack in action
+    documentation: https://docs.stackable.tech/stackablectl/stable/demos/logging.html
+    stackableStack: logging
+    labels:
+      - logging
+      - opensearch
+      - opensearch-dashboards
+      - vector
+      - zookeeper
+    manifests:
+      - plainYaml: https://github.com/stackabletech/stackablectl/main/demos/logging/zookeeper.yaml

demos/logging/zookeeper.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperCluster
+metadata:
+  name: simple-zk
+spec:
+  image:
+    productVersion: 3.8.0
+    stackableVersion: 0.9.0
+  vectorAggregatorConfigMapName: vector-aggregator-discovery
+  servers:
+    roleGroups:
+      default:
+        replicas: 3
+        config:
+          logging:
+            enableVectorAgent: true
+            containers:
+              vector:
+                file:
+                  level: WARN
+              zookeeper:
+                console:
+                  level: INFO
+                file:
+                  level: INFO
+                loggers:
+                  ROOT:
+                    level: INFO
+                  org.apache.zookeeper.server.NettyServerCnxn:
+                    level: NONE

docs/modules/ROOT/nav.adoc

@@ -10,6 +10,7 @@
 ** xref:demos/airflow-scheduled-job.adoc[]
 ** xref:demos/data-lakehouse-iceberg-trino-spark.adoc[]
 ** xref:demos/hbase-hdfs-load-cycling-data.adoc[]
+** xref:demos/logging.adoc[]
 ** xref:demos/nifi-kafka-druid-earthquake-data.adoc[]
 ** xref:demos/nifi-kafka-druid-water-level-data.adoc[]
 ** xref:demos/spark-k8s-anomaly-detection-taxi-data.adoc[]

docs/modules/ROOT/pages/demos/logging.adoc

@@ -0,0 +1,110 @@
+= logging
+
+This demo will
+
+* Install the required Stackable operators.
+* Spin up the follow data products:
+** *Apache ZooKeeper*: A centralized service for maintaining configuration
+information, naming, providing distributed synchronization, and providing group
+services. This demo makes its log data observable in OpenSearch Dashboards.
+** *Vector*: A tool for building observability pipelines. This demo uses Vector
+as a log agent to gather and transform the logs and as an aggregator to forward
+the collected logs to OpenSearch.
+** *OpenSearch*: A data store and search engine. This demo uses it to store and
+index the log data.
+** *OpenSearch Dashboards*: A visualization and user interface. This demo uses
+it to make the log data easily accessible to the user.
+* Create a view in OpenSearch Dashboards for convenient browsing the log data.
+
+You can see the deployed products as well as their relationship in the
+following diagram:
+
+image::logging/overview.png[]
+
+== OpenSearch prerequisites
+
+=== Mac and Windows
+
+If you are running on Mac or Windows and use Docker to run Kubernetes then set
+the RAM to at least 4 GB in _Preferences_ > _Resources_.
+
+=== Linux
+
+OpenSearch uses a mmapfs directory by default to store its indices. The default
+operating system limits on mmap counts is likely to be too low – usually 65530,
+which may result in out of memory exceptions. So the Linux setting
+`vm.max_map_count` on the host machine where kind is running, must be set to at
+least 262144.
+
+To check the current value, run this command:
+
+[source,console]
+----
+sysctl vm.max_map_count
+----
+
+The limit can be temporarily increased with:
+
+[source,console]
+----
+sudo sysctl --write vm.max_map_count=262144
+----
+
+To permanently increase the value, add the following line to `/etc/sysctl.conf`:
+
+[source,.properties]
+----
+vm.max_map_count=262144
+----
+
+Then run `sudo sysctl --load` to reload.
+
+== Run the demo
+
+The following command creates a kind cluster and installs this demo:
+
+[source,console]
+----
+$ stackablectl demo install logging --kind-cluster
+----
+
+== List deployed Stackable services
+
+To list the installed Stackable services run the following command:
+
+[source,console]
+----
+$ stackablectl services list
+┌───────────────────────┬───────────────────────┬───────────┬──────────────────────────────┬───────────────────────────────────────────────────────────────────────────────────┐
+│ Product               ┆ Name                  ┆ Namespace ┆ Endpoints                    ┆ Info                                                                              │
+╞═══════════════════════╪═══════════════════════╪═══════════╪══════════════════════════════╪═══════════════════════════════════════════════════════════════════════════════════╡
+│ opensearch-dashboards ┆ opensearch-dashboards ┆ default   ┆ http http://172.18.0.5:31319 ┆ Third party service                                                               │
+│                       ┆                       ┆           ┆                              ┆ Logs view: http://172.18.0.5:31319/app/discover?security_tenant=global#/view/logs │
+│                       ┆                       ┆           ┆                              ┆ Username: admin, password: adminadmin                                             │
+├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ zookeeper             ┆ simple-zk             ┆ default   ┆ zk   172.18.0.2:32417        ┆                                                                                   │
+└───────────────────────┴───────────────────────┴───────────┴──────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────┘
+----
+
+[NOTE]
+====
+When a product instance has not finished starting yet, the service will have no
+endpoint. Starting all the product instances might take a considerable amount
+of time depending on your internet connectivity. In case the product is not
+ready yet, a warning might be shown.
+====
+
+== Inspect the log data
+
+You can have a look at the log data within the OpenSearch Dashboards web
+interface by running `stackablectl services list` and opening the URL shown in
+the info column of the `opensearch-dashboards` entry. In this case it is
+http://172.18.0.5:31319/app/discover?security_tenant=global#/view/logs.
+
+image::logging/login.png[]
+
+Log in with the username `admin` and password `adminadmin`.
+
+image::logging/logs.png[]
+
+Inspect the logs.

src/services.rs

@@ -176,6 +176,10 @@ async fn list_services(
         "minio".to_string(),
         get_minio_services(!all_namespaces, redact_credentials).await?,
     );
+    output.insert(
+        "opensearch-dashboards".to_string(),
+        get_opensearch_dashboards_services(!all_namespaces, redact_credentials).await?,
+    );
 
     match output_type {
         OutputType::Text => {
@@ -188,7 +192,7 @@ async fn list_services(
                     Cell::new("Name"),
                     Cell::new("Namespace"),
                     Cell::new("Endpoints"),
-                    Cell::new("Extra infos"),
+                    Cell::new("Info"),
                 ]);
 
             let max_endpoint_name_length = output
@@ -216,7 +220,7 @@ async fn list_services(
                         .join("\n");
 
                     table.add_row(vec![
-                        Cell::new(&product_name),
+                        Cell::new(product_name.as_str()),
                         Cell::new(installed_product.name),
                         Cell::new(installed_product.namespace.unwrap_or_default()),
                         Cell::new(endpoints),
@@ -551,3 +555,74 @@ pub async fn get_minio_service(
         extra_infos,
     })
 }
+
+async fn get_opensearch_dashboards_services(
+    namespaced: bool,
+    redact_credentials: bool,
+) -> Result<Vec<InstalledProduct>, Box<dyn Error>> {
+    let client = get_client().await?;
+    let list_params = ListParams::default().labels("app.kubernetes.io/name=opensearch-dashboards");
+
+    let mut result = Vec::new();
+
+    let service_api: Api<Service> = if namespaced {
+        Api::namespaced(client.to_owned(), NAMESPACE.lock()?.as_str())
+    } else {
+        Api::all(client.clone())
+    };
+
+    let services = service_api.list(&list_params).await?;
+    for service in services {
+        let namespace = service.namespace().unwrap();
+        let endpoints =
+            get_service_endpoint_urls(&service, &service.name_unchecked(), client.to_owned())
+                .await?;
+
+        let annotations = service.annotations();
+
+        let mut extra_infos = vec!["Third party service".to_string()];
+
+        if let Some(http_endpoint) = endpoints.get("http") {
+            if let Some(logs_endpoint) = annotations.get("stackable.tech/logging-view-logs") {
+                extra_infos.push(format!("Logs view: {http_endpoint}{logs_endpoint}"));
+            }
+        }
+
+        if let Some(credentials_secret) =
+            annotations.get("stackable.tech/logging-credentials-secret")
+        {
+            let secret_api: Api<Secret> = Api::namespaced(client.clone(), &namespace);
+            if let Ok(credentials) = secret_api.get(credentials_secret).await {
+                if let Some(username) = credentials
+                    .data
+                    .as_ref()
+                    .and_then(|data| data.get("username"))
+                    .and_then(|value| String::from_utf8(value.0.to_owned()).ok())
+                {
+                    let mut credentials_info = format!("Username: {username}");
+                    if redact_credentials {
+                        credentials_info.push_str(&format!(", password: {REDACTED_PASSWORD}"));
+                    } else if let Some(password) = credentials
+                        .data
+                        .as_ref()
+                        .and_then(|data| data.get("password"))
+                        .and_then(|value| String::from_utf8(value.0.to_owned()).ok())
+                    {
+                        credentials_info.push_str(&format!(", password: {password}"));
+                    }
+                    extra_infos.push(credentials_info);
+                }
+            }
+        }
+
+        let installed_product = InstalledProduct {
+            name: service.name_unchecked(),
+            namespace: service.namespace(),
+            endpoints,
+            extra_infos,
+        };
+        result.push(installed_product);
+    }
+
+    Ok(result)
+}

stacks/logging/vector-aggregator-discovery.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+data:
+  ADDRESS: vector-aggregator:6000