metricsProps = new HashMap<>();
+
+ // defines the metrics update period, default is 60 seconds.
+ final LoggingRegistryConfig lconf = c -> {
+ if (c.equals("logging.step")) {
+ return metricsProps.getOrDefault("logging.step", "60s");
+ }
+ return null;
+ };
+
+ LOG.info("Creating logging metrics registry with params: {}", params);
+ metricsProps.putAll(params.getOptions());
+ return LoggingMeterRegistry.builder(lconf).loggingSink(metricConsumer).build();
+ }
+}
diff --git a/core/src/main/java/org/apache/accumulo/core/spi/metrics/MeterRegistryFactory.java b/core/src/main/java/org/apache/accumulo/core/spi/metrics/MeterRegistryFactory.java
new file mode 100644
index 00000000000..3e57cc200a1
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/spi/metrics/MeterRegistryFactory.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.spi.metrics;
+
+import java.util.Map;
+
+import org.apache.accumulo.core.spi.common.ServiceEnvironment;
+
+import io.micrometer.core.instrument.MeterRegistry;
+
+/**
+ * The Micrometer metrics allows for different monitoring systems. and can be enabled within
+ * Accumulo with properties and are initialized by implementing this interface and providing the
+ * factory implementation clas name as a property. Metrics are specified with the following
+ * properties:
+ *
+ * Property.GENERAL_MICROMETER_ENABLED = true
+ *
+ * Property.GENERAL_MICROMETER_FACTORY = [implementation].class.getName()
+ *
+ * @since 2.1.3
+ */
+public interface MeterRegistryFactory {
+ // full form in property file is "general.custom.metrics.opts"
+ String METRICS_PROP_SUBSTRING = "metrics.opts.";
+
+ interface InitParameters {
+ /**
+ * Get the configured metrics properties passed as {@code general.custom.metrics.opts} The
+ * returned map is the stripped names with {@code general.custom.metrics.opts} removed.
+ *
+ * For example properties {@code general.custom.metrics.opts.prop1=abc} and
+ * {@code general.custom.metrics.opts.prop9=123} are set, then this map would contain
+ * {@code prop1=abc} and {@code prop9=123}.
+ *
+ * @return a map of property name, value pairs, stripped of a prefix.
+ */
+ Map getOptions();
+
+ /**
+ * Optional extension point to pass additional information though the ServiceEnvironment.
+ *
+ * @return the service environment
+ */
+ ServiceEnvironment getServiceEnv();
+ }
+
+ /**
+ * Called on metrics initialization. Implementations should note the initial parameters set when
+ * instantiating a MeterRegistry should be considered fixed. Once a MeterRegistry is initialized
+ * parameters such as common tags may not be updated with later additions or changes.
+ *
+ * @return a Micrometer registry that will be added to the metrics configuration.
+ */
+ MeterRegistry create(final InitParameters params);
+}
diff --git a/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java b/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
index 147f19e5dd3..ea26c563e41 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
@@ -31,6 +31,7 @@
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledFuture;
@@ -42,7 +43,6 @@
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.trace.TraceUtil;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.slf4j.Logger;
@@ -51,6 +51,8 @@
import com.google.common.base.Preconditions;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics;
@SuppressFBWarnings(value = "RV_EXCEPTION_NOT_THROWN",
justification = "Throwing Error for it to be caught by AccumuloUncaughtExceptionHandler")
@@ -465,16 +467,30 @@ public ThreadPoolExecutorBuilder atPriority(@NonNull final OptionalInt priority)
}
/**
- * When set to true will emit metrics and register the metrics in a static registry. After the
- * thread pool is deleted, there will still be metrics objects related to it in the static
- * registry. There is no way to clean these leftover objects up therefore its recommended that
- * this option only be set true for long-lived thread pools. Creating lots of short-lived thread
- * pools and registering them can lead to out of memory errors over long time periods.
+ * When set to true will emit metrics and register the metrics in a registry. After the thread
+ * pool is deleted, there will still be metrics objects related to it in the static registry.
+ * There is no way to clean these leftover objects up therefore its recommended that this option
+ * only be set true for long-lived thread pools. Creating lots of short-lived thread pools and
+ * registering them can lead to out of memory errors over long time periods.
*
* @return a fluent-style builder instance
*/
public ThreadPoolExecutorBuilder enableThreadPoolMetrics() {
- this.emitThreadPoolMetrics = true;
+ return enableThreadPoolMetrics(true);
+ }
+
+ /**
+ * Optionally set to register pool metrics. When set to true will emit metrics and register the
+ * metrics in a registry. After the thread pool is deleted, there will still be metrics objects
+ * related to it in the static registry. There is no way to clean these leftover objects up
+ * therefore its recommended that this option only be set true for long-lived thread pools.
+ * Creating lots of short-lived thread pools and registering them can lead to out of memory
+ * errors over long time periods.
+ *
+ * @return a fluent-style builder instance
+ */
+ public ThreadPoolExecutorBuilder enableThreadPoolMetrics(final boolean enable) {
+ this.emitThreadPoolMetrics = enable;
return this;
}
}
@@ -539,7 +555,7 @@ public Future submit(@NonNull Runnable task) {
result.allowCoreThreadTimeOut(true);
}
if (emitThreadPoolMetrics) {
- MetricsUtil.addExecutorServiceMetrics(result, name);
+ ThreadPools.addExecutorServiceMetrics(result, name);
}
return result;
}
@@ -646,9 +662,13 @@ public boolean remove(Runnable task) {
};
if (emitThreadPoolMetrics) {
- MetricsUtil.addExecutorServiceMetrics(result, name);
+ ThreadPools.addExecutorServiceMetrics(result, name);
}
return result;
}
+ private static void addExecutorServiceMetrics(ExecutorService executor, String name) {
+ new ExecutorServiceMetrics(executor, name, List.of()).bindTo(Metrics.globalRegistry);
+ }
+
}
diff --git a/core/src/test/java/org/apache/accumulo/core/spi/metrics/LoggingMeterRegistryFactoryTest.java b/core/src/test/java/org/apache/accumulo/core/spi/metrics/LoggingMeterRegistryFactoryTest.java
new file mode 100644
index 00000000000..9d628d27752
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/spi/metrics/LoggingMeterRegistryFactoryTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.spi.metrics;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+import java.util.Map;
+
+import org.apache.accumulo.core.spi.common.ServiceEnvironment;
+import org.junit.jupiter.api.Test;
+
+import io.micrometer.core.instrument.logging.LoggingMeterRegistry;
+
+class LoggingMeterRegistryFactoryTest {
+
+ @Test
+ public void createTest() {
+ LoggingMeterRegistryFactory factory = new LoggingMeterRegistryFactory();
+ var reg = factory.create(new LoggingMetricsParams());
+ assertInstanceOf(LoggingMeterRegistry.class, reg);
+ }
+
+ private static class LoggingMetricsParams implements MeterRegistryFactory.InitParameters {
+
+ @Override
+ public Map getOptions() {
+ // note: general.custom.metrics.opts. is expected to be stripped before passing the options.
+ return Map.of("prop1", "abc", "logging.step", "1s");
+ }
+
+ @Override
+ public ServiceEnvironment getServiceEnv() {
+ return null;
+ }
+ }
+}
diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java
index 9aac388e748..025b320f74d 100644
--- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java
@@ -147,6 +147,9 @@ MiniAccumuloConfigImpl initialize() {
mergeProp(Property.INSTANCE_SECRET.getKey(), DEFAULT_INSTANCE_SECRET);
}
+ // enable metrics reporting - by default will appear in standard log files.
+ mergeProp(Property.GENERAL_MICROMETER_ENABLED.getKey(), "true");
+
mergeProp(Property.TSERV_PORTSEARCH.getKey(), "true");
mergeProp(Property.TSERV_DATACACHE_SIZE.getKey(), "10M");
mergeProp(Property.TSERV_INDEXCACHE_SIZE.getKey(), "10M");
diff --git a/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java b/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java
index a23ba911db2..ac1bcaab90e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java
@@ -24,7 +24,6 @@
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.classloader.ClassLoaderUtil;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.server.security.SecurityUtil;
import org.slf4j.Logger;
@@ -89,9 +88,11 @@ public AccumuloConfiguration getConfiguration() {
return getContext().getConfiguration();
}
- @Override
- public void close() {
- MetricsUtil.close();
+ public String getApplicationName() {
+ return applicationName;
}
+ @Override
+ public void close() {}
+
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java
index 58947612569..4366967c2e3 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java
@@ -53,6 +53,7 @@
import org.apache.accumulo.core.fate.zookeeper.ZooReader;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.rpc.SslConnectionParams;
import org.apache.accumulo.core.singletons.SingletonReservation;
import org.apache.accumulo.core.spi.crypto.CryptoServiceFactory;
@@ -67,6 +68,7 @@
import org.apache.accumulo.server.conf.store.impl.ZooPropStore;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.metadata.ServerAmpleImpl;
+import org.apache.accumulo.server.metrics.MetricsInfoImpl;
import org.apache.accumulo.server.rpc.SaslServerConnectionParams;
import org.apache.accumulo.server.rpc.ThriftServerType;
import org.apache.accumulo.server.security.AuditedSecurityOperation;
@@ -101,6 +103,7 @@ public class ServerContext extends ClientContext {
private final Supplier sharedScheduledThreadPool;
private final Supplier securityOperation;
private final Supplier cryptoFactorySupplier;
+ private final Supplier metricsInfoSupplier;
public ServerContext(SiteConfiguration siteConfig) {
this(new ServerInfo(siteConfig));
@@ -125,6 +128,7 @@ private ServerContext(ServerInfo info) {
securityOperation =
memoize(() -> new AuditedSecurityOperation(this, SecurityOperation.getAuthorizor(this),
SecurityOperation.getAuthenticator(this), SecurityOperation.getPermHandler(this)));
+ metricsInfoSupplier = memoize(() -> new MetricsInfoImpl(this));
}
/**
@@ -445,4 +449,13 @@ public AuditedSecurityOperation getSecurityOperation() {
return securityOperation.get();
}
+ public MetricsInfo getMetricsInfo() {
+ return metricsInfoSupplier.get();
+ }
+
+ @Override
+ public void close() {
+ getMetricsInfo().close();
+ super.close();
+ }
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImpl.java b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImpl.java
index 5c42d0c0a3d..d55c9465b16 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImpl.java
@@ -48,13 +48,10 @@ public class PropCacheCaffeineImpl implements PropCache {
ThreadPools.getServerThreadPools().getPoolBuilder("caffeine-tasks").numCoreThreads(1)
.numMaxThreads(20).withTimeOut(60L, SECONDS).build();
- private final PropStoreMetrics metrics;
-
private final LoadingCache,VersionedProperties> cache;
private PropCacheCaffeineImpl(final CacheLoader,VersionedProperties> cacheLoader,
- final PropStoreMetrics metrics, final Ticker ticker, boolean runTasksInline) {
- this.metrics = metrics;
+ final Ticker ticker, boolean runTasksInline) {
var builder = Caffeine.newBuilder().expireAfterAccess(EXPIRE_MIN, BASE_TIME_UNITS)
.evictionListener(this::evictionNotifier);
if (runTasksInline) {
@@ -68,14 +65,9 @@ private PropCacheCaffeineImpl(final CacheLoader,VersionedPropert
cache = builder.build(cacheLoader);
}
- public PropStoreMetrics getMetrics() {
- return metrics;
- }
-
void evictionNotifier(PropStoreKey propStoreKey, VersionedProperties value,
RemovalCause cause) {
log.trace("Evicted: ID: {} was evicted from cache. Reason: {}", propStoreKey, cause);
- metrics.incrEviction();
}
@Override
@@ -85,7 +77,6 @@ void evictionNotifier(PropStoreKey propStoreKey, VersionedProperties value,
return cache.get(propStoreKey);
} catch (Exception ex) {
log.info("Cache failed to retrieve properties for: " + propStoreKey, ex);
- metrics.incrZkError();
return null;
}
}
@@ -116,20 +107,17 @@ public void removeAll() {
}
public static class Builder {
-
- private final PropStoreMetrics metrics;
private final ZooPropLoader zooPropLoader;
private Ticker ticker = null;
private boolean runTasksInline = false;
- public Builder(final ZooPropLoader zooPropLoader, final PropStoreMetrics metrics) {
+ public Builder(final ZooPropLoader zooPropLoader) {
Objects.requireNonNull(zooPropLoader, "A PropStoreChangeMonitor must be provided");
this.zooPropLoader = zooPropLoader;
- this.metrics = metrics;
}
public PropCacheCaffeineImpl build() {
- return new PropCacheCaffeineImpl(zooPropLoader, metrics, ticker, runTasksInline);
+ return new PropCacheCaffeineImpl(zooPropLoader, ticker, runTasksInline);
}
public Builder forTests(final Ticker ticker) {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropStoreMetrics.java b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropStoreMetrics.java
deleted file mode 100644
index a85c14eec21..00000000000
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropStoreMetrics.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.conf.store.impl;
-
-import java.time.Duration;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.metrics.MetricsProducer;
-import org.apache.accumulo.core.metrics.MetricsUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.MeterRegistry;
-import io.micrometer.core.instrument.Timer;
-
-public class PropStoreMetrics implements MetricsProducer {
-
- private static final Logger log = LoggerFactory.getLogger(PropStoreMetrics.class);
-
- private Timer load;
- private Counter refresh;
- private Counter refreshLoad;
- private Counter eviction;
- private Counter zkError;
-
- @Override
- public void registerMetrics(MeterRegistry registry) {
-
- load = Timer.builder(METRICS_PROPSTORE_LOAD_TIMER).description("prop store load time")
- .tags(MetricsUtil.getCommonTags()).register(registry);
-
- refresh =
- Counter.builder(METRICS_PROPSTORE_REFRESH_COUNT).description("prop store refresh count")
- .tags(MetricsUtil.getCommonTags()).register(registry);
-
- refreshLoad = Counter.builder(METRICS_PROPSTORE_REFRESH_LOAD_COUNT)
- .description("prop store refresh load count").tags(MetricsUtil.getCommonTags())
- .register(registry);
-
- eviction =
- Counter.builder(METRICS_PROPSTORE_EVICTION_COUNT).description("prop store eviction count")
- .tags(MetricsUtil.getCommonTags()).register(registry);
-
- zkError = Counter.builder(METRICS_PROPSTORE_ZK_ERROR_COUNT)
- .description("prop store ZooKeeper error count").tags(MetricsUtil.getCommonTags())
- .register(registry);
-
- }
-
- public PropStoreMetrics() {
- log.debug("Creating PropStore metrics");
- }
-
- public void addLoadTime(final long value) {
- log.trace("Load time: {}", value);
- load.record(Duration.ofMillis(value));
- log.trace("Load count: {} time:{}", load.count(), load.totalTime(TimeUnit.MILLISECONDS));
- }
-
- public void incrRefresh() {
- refresh.increment();
- }
-
- public void incrRefreshLoad() {
- refreshLoad.increment();
- }
-
- public void incrEviction() {
- eviction.increment();
- }
-
- public void incrZkError() {
- zkError.increment();
- }
-}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropLoader.java b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropLoader.java
index 48620f6a924..c5842e5b663 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropLoader.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropLoader.java
@@ -22,7 +22,6 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.server.conf.codec.VersionedPropCodec;
@@ -44,14 +43,12 @@ public class ZooPropLoader implements CacheLoader,VersionedPrope
private final VersionedPropCodec propCodec;
// used to set watcher, does not react to events.
private final PropStoreWatcher propStoreWatcher;
- private final PropStoreMetrics metrics;
public ZooPropLoader(final ZooReaderWriter zrw, final VersionedPropCodec propCodec,
- final PropStoreWatcher propStoreWatcher, final PropStoreMetrics metrics) {
+ final PropStoreWatcher propStoreWatcher) {
this.zrw = zrw;
this.propCodec = propCodec;
this.propStoreWatcher = propStoreWatcher;
- this.metrics = metrics;
}
@Override
@@ -59,26 +56,18 @@ public ZooPropLoader(final ZooReaderWriter zrw, final VersionedPropCodec propCod
try {
log.trace("load called for {}", propStoreKey);
- long startNanos = System.nanoTime();
-
Stat stat = new Stat();
byte[] bytes = zrw.getData(propStoreKey.getPath(), propStoreWatcher, stat);
if (stat.getDataLength() == 0) {
return new VersionedProperties();
}
VersionedProperties vProps = propCodec.fromBytes(stat.getVersion(), bytes);
-
- metrics.addLoadTime(
- TimeUnit.MILLISECONDS.convert(System.nanoTime() - startNanos, TimeUnit.NANOSECONDS));
-
return vProps;
} catch (KeeperException.NoNodeException ex) {
- metrics.incrZkError();
log.debug("property node for {} does not exist - it may be being created", propStoreKey);
propStoreWatcher.signalZkChangeEvent(propStoreKey);
return null;
} catch (Exception ex) {
- metrics.incrZkError();
log.info("Failed to load properties for: {} from ZooKeeper, returning null", propStoreKey,
ex);
propStoreWatcher.signalZkChangeEvent(propStoreKey);
@@ -97,8 +86,6 @@ public CompletableFuture asyncLoad(PropStoreKey asyncReload(PropStoreKey propStoreKey,
VersionedProperties oldValue, Executor executor) throws Exception {
log.trace("asyncReload called for key: {}", propStoreKey);
- metrics.incrRefresh();
-
return CompletableFuture.supplyAsync(() -> loadIfDifferentVersion(propStoreKey, oldValue),
executor);
}
@@ -107,7 +94,6 @@ public CompletableFuture asyncReload(PropStoreKey propSt
public @Nullable VersionedProperties reload(PropStoreKey propStoreKey,
VersionedProperties oldValue) throws Exception {
log.trace("reload called for: {}", propStoreKey);
- metrics.incrRefresh();
return loadIfDifferentVersion(propStoreKey, oldValue);
}
@@ -141,8 +127,6 @@ public CompletableFuture asyncReload(PropStoreKey propSt
var updatedValue = load(propCacheId);
- metrics.incrRefreshLoad();
-
// The cache will be updated - notify external listeners value changed.
propStoreWatcher.signalCacheChangeEvent(propCacheId);
log.trace("Updated value {}", updatedValue == null ? "null" : updatedValue.print(true));
@@ -150,7 +134,6 @@ public CompletableFuture asyncReload(PropStoreKey propSt
} catch (RuntimeException | KeeperException | InterruptedException ex) {
log.warn("async exception occurred reading properties from ZooKeeper for: {} returning null",
propCacheId, ex);
- metrics.incrZkError();
propStoreWatcher.signalZkChangeEvent(propCacheId);
return null;
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
index a345b58c468..d25cbd2bd49 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
@@ -31,7 +31,6 @@
import org.apache.accumulo.core.fate.zookeeper.ZooReader;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.server.conf.codec.VersionedPropCodec;
import org.apache.accumulo.server.conf.codec.VersionedProperties;
import org.apache.accumulo.server.conf.store.PropCache;
@@ -57,7 +56,6 @@ public class ZooPropStore implements PropStore, PropChangeListener {
private final ZooReaderWriter zrw;
private final PropStoreWatcher propStoreWatcher;
private final PropCacheCaffeineImpl cache;
- private final PropStoreMetrics cacheMetrics = new PropStoreMetrics();
private final ReadyMonitor zkReadyMon;
/**
@@ -91,17 +89,14 @@ private ZooPropStore(final InstanceId instanceId, final ZooReaderWriter zrw) {
this.propStoreWatcher = requireNonNullElseGet(watcher, () -> new PropStoreWatcher(zkReadyMon));
- ZooPropLoader propLoader = new ZooPropLoader(zrw, codec, this.propStoreWatcher, cacheMetrics);
+ ZooPropLoader propLoader = new ZooPropLoader(zrw, codec, this.propStoreWatcher);
if (ticker == null) {
- this.cache = new PropCacheCaffeineImpl.Builder(propLoader, cacheMetrics).build();
+ this.cache = new PropCacheCaffeineImpl.Builder(propLoader).build();
} else {
- this.cache =
- new PropCacheCaffeineImpl.Builder(propLoader, cacheMetrics).forTests(ticker).build();
+ this.cache = new PropCacheCaffeineImpl.Builder(propLoader).forTests(ticker).build();
}
- MetricsUtil.initializeProducers(cacheMetrics);
-
try {
var path = ZooUtil.getRoot(instanceId);
if (zrw.exists(path, propStoreWatcher)) {
@@ -141,10 +136,6 @@ public boolean exists(final PropStoreKey propStoreKey) {
return false;
}
- public PropStoreMetrics getMetrics() {
- return cacheMetrics;
- }
-
@Override
public void create(PropStoreKey propStoreKey, Map props) {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImpl.java
new file mode 100644
index 00000000000..f3b447defeb
--- /dev/null
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImpl.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import static org.apache.accumulo.core.conf.Property.GENERAL_ARBITRARY_PROP_PREFIX;
+import static org.apache.accumulo.core.spi.metrics.MeterRegistryFactory.METRICS_PROP_SUBSTRING;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.accumulo.core.spi.common.ServiceEnvironment;
+import org.apache.accumulo.core.spi.metrics.MeterRegistryFactory;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.accumulo.server.ServiceEnvironmentImpl;
+
+/**
+ * Provides a way to pass parameters from an Accumulo configuration to the MeterRegistryFactory.
+ * Properties need have the form {@code general.custom.metrics.opts.PARAMETER_NAME = VALUE}. The
+ * prefix {@code general.custom.metrics.opts.} is stripped and the resulting Map returned by
+ * {@link #getOptions()} will be map of {@code PROPERTY_NAME, VALUE} key value pairs.
+ *
+ * Other implementations can extend
+ * {@link org.apache.accumulo.core.spi.metrics.MeterRegistryFactory.InitParameters} to provide other
+ * implementations
+ */
+public class MeterRegistryEnvPropImpl implements MeterRegistryFactory.InitParameters {
+
+ private final ServerContext context;
+
+ public MeterRegistryEnvPropImpl(final ServerContext context) {
+ this.context = context;
+ }
+
+ /**
+ * Properties that match {@code general.custom.metrics.opts.PARAMETER_NAME = VALUE} with be
+ * filtered and returned with the prefix stripped.
+ *
+ * @return a map of the filtered, stripped property, value pairs.
+ */
+ @Override
+ public Map getOptions() {
+ Map filtered = new HashMap<>();
+
+ Map options = context.getConfiguration()
+ .getAllPropertiesWithPrefixStripped(GENERAL_ARBITRARY_PROP_PREFIX);
+ options.forEach((k, v) -> {
+ if (k.startsWith(METRICS_PROP_SUBSTRING)) {
+ String name = k.substring(METRICS_PROP_SUBSTRING.length());
+ filtered.put(name, v);
+ }
+ });
+ return filtered;
+ }
+
+ @Override
+ public ServiceEnvironment getServiceEnv() {
+ return new ServiceEnvironmentImpl(context);
+ }
+}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/metrics/MetricsInfoImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metrics/MetricsInfoImpl.java
new file mode 100644
index 00000000000..15322bc495c
--- /dev/null
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/MetricsInfoImpl.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import static org.apache.hadoop.util.StringUtils.getTrimmedStrings;
+
+import java.lang.reflect.InvocationTargetException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.accumulo.core.classloader.ClassLoaderUtil;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.metrics.MetricsInfo;
+import org.apache.accumulo.core.metrics.MetricsProducer;
+import org.apache.accumulo.core.util.HostAndPort;
+import org.apache.accumulo.server.ServerContext;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.micrometer.core.instrument.Meter;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Tag;
+import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics;
+import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;
+import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
+import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
+import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
+import io.micrometer.core.instrument.composite.CompositeMeterRegistry;
+import io.micrometer.core.instrument.config.MeterFilter;
+import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
+
+public class MetricsInfoImpl implements MetricsInfo {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MetricsInfoImpl.class);
+
+ private final ServerContext context;
+
+ private final Lock lock = new ReentrantLock();
+
+ private final Map commonTags;
+
+ // JvmGcMetrics are declared with AutoCloseable - keep reference to use with close()
+ private JvmGcMetrics jvmGcMetrics;
+
+ private final boolean metricsEnabled;
+
+ private CompositeMeterRegistry composite = null;
+ private final List pendingRegistries = new ArrayList<>();
+
+ private final List producers = new ArrayList<>();
+
+ public MetricsInfoImpl(final ServerContext context) {
+ this.context = context;
+ metricsEnabled = context.getConfiguration().getBoolean(Property.GENERAL_MICROMETER_ENABLED);
+ printMetricsConfig();
+ commonTags = new HashMap<>();
+ Tag t = MetricsInfo.instanceNameTag(context.getInstanceName());
+ commonTags.put(t.getKey(), t);
+ }
+
+ private void printMetricsConfig() {
+ final boolean jvmMetricsEnabled =
+ context.getConfiguration().getBoolean(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED);
+ LOG.info("micrometer metrics enabled: {}", metricsEnabled);
+ if (jvmMetricsEnabled) {
+ if (metricsEnabled) {
+ LOG.info("detailed jvm metrics enabled: {}", jvmMetricsEnabled);
+ } else {
+ LOG.info("requested jvm metrics, but micrometer metrics are disabled.");
+ }
+ }
+ if (metricsEnabled) {
+ LOG.info("metrics registry factories: {}",
+ context.getConfiguration().get(Property.GENERAL_MICROMETER_FACTORY));
+ }
+ }
+
+ @Override
+ public boolean isMetricsEnabled() {
+ return metricsEnabled;
+ }
+
+ /**
+ * Common tags for all services.
+ */
+ @Override
+ public void addServiceTags(final String applicationName, final HostAndPort hostAndPort) {
+ List tags = new ArrayList<>();
+
+ if (applicationName != null && !applicationName.isEmpty()) {
+ tags.add(MetricsInfo.processTag(applicationName));
+ }
+ if (hostAndPort != null) {
+ tags.addAll(MetricsInfo.addressTags(hostAndPort));
+ }
+ addCommonTags(tags);
+ }
+
+ @Override
+ public void addCommonTags(List updates) {
+ lock.lock();
+ try {
+ if (composite != null) {
+ LOG.warn(
+ "Common tags after registry has been initialized may be ignored. Current common tags: {}",
+ commonTags);
+ return;
+ }
+ updates.forEach(t -> commonTags.put(t.getKey(), t));
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public Collection getCommonTags() {
+ lock.lock();
+ try {
+ return Collections.unmodifiableCollection(commonTags.values());
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void addRegistry(MeterRegistry registry) {
+ lock.lock();
+ try {
+ if (composite != null) {
+ composite.add(registry);
+ } else {
+ // defer until composite is initialized
+ pendingRegistries.add(registry);
+ }
+
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void addMetricsProducers(MetricsProducer... producer) {
+ if (producer.length == 0) {
+ LOG.debug(
+ "called addMetricsProducers() without providing at least one producer - this has no effect");
+ return;
+ }
+ lock.lock();
+ try {
+ if (composite == null) {
+ producers.addAll(Arrays.asList(producer));
+ } else {
+ Arrays.stream(producer).forEach(p -> p.registerMetrics(composite));
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public MeterRegistry getRegistry() {
+ lock.lock();
+ try {
+ if (composite == null) {
+ throw new IllegalStateException("metrics have not been initialized, call init() first");
+ }
+ } finally {
+ lock.unlock();
+ }
+ return composite;
+ }
+
+ @Override
+ public void init() {
+ lock.lock();
+ try {
+ if (composite != null) {
+ LOG.warn("metrics registry has already been initialized");
+ return;
+ }
+ composite = new CompositeMeterRegistry();
+ composite.config().commonTags(commonTags.values());
+
+ LOG.info("Metrics initialization. common tags: {}", commonTags);
+
+ boolean jvmMetricsEnabled =
+ context.getConfiguration().getBoolean(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED);
+
+ if (jvmMetricsEnabled) {
+ LOG.info("enabling detailed jvm, classloader, jvm gc and process metrics");
+ new ClassLoaderMetrics().bindTo(composite);
+ new JvmMemoryMetrics().bindTo(composite);
+ jvmGcMetrics = new JvmGcMetrics();
+ jvmGcMetrics.bindTo(composite);
+ new ProcessorMetrics().bindTo(composite);
+ new JvmThreadMetrics().bindTo(composite);
+ }
+
+ MeterFilter replicationFilter = new MeterFilter() {
+ @Override
+ public DistributionStatisticConfig configure(Meter.Id id,
+ @NonNull DistributionStatisticConfig config) {
+ if (id.getName().equals("replicationQueue")) {
+ return DistributionStatisticConfig.builder().percentiles(0.5, 0.75, 0.9, 0.95, 0.99)
+ .expiry(Duration.ofMinutes(10)).build().merge(config);
+ }
+ return config;
+ }
+ };
+
+ if (isMetricsEnabled()) {
+ // user specified registries
+ String userRegistryFactories =
+ context.getConfiguration().get(Property.GENERAL_MICROMETER_FACTORY);
+
+ for (String factoryName : getTrimmedStrings(userRegistryFactories)) {
+ try {
+ MeterRegistry registry = getRegistryFromFactory(factoryName, context);
+ registry.config().commonTags(commonTags.values());
+ registry.config().meterFilter(replicationFilter);
+ addRegistry(registry);
+ } catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException
+ | InstantiationException | IllegalAccessException ex) {
+ LOG.warn("Could not load registry {}", factoryName, ex);
+ }
+ }
+ }
+
+ pendingRegistries.forEach(registry -> composite.add(registry));
+
+ LOG.info("Metrics initialization. Register producers: {}", producers);
+ producers.forEach(p -> p.registerMetrics(composite));
+
+ Metrics.globalRegistry.add(composite);
+
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @VisibleForTesting
+ @SuppressWarnings({"deprecation",
+ "support for org.apache.accumulo.core.metrics.MeterRegistryFactory can be removed in 3.1"})
+ static MeterRegistry getRegistryFromFactory(final String factoryName, final ServerContext context)
+ throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException,
+ InstantiationException, IllegalAccessException {
+ try {
+ LOG.info("look for meter spi registry factory {}", factoryName);
+ Class clazz =
+ ClassLoaderUtil.loadClass(factoryName,
+ org.apache.accumulo.core.spi.metrics.MeterRegistryFactory.class);
+ org.apache.accumulo.core.spi.metrics.MeterRegistryFactory factory =
+ clazz.getDeclaredConstructor().newInstance();
+ org.apache.accumulo.core.spi.metrics.MeterRegistryFactory.InitParameters initParameters =
+ new MeterRegistryEnvPropImpl(context);
+ return factory.create(initParameters);
+ } catch (ClassCastException ex) {
+ // empty. On exception try deprecated version
+ }
+ try {
+ LOG.info("find legacy meter registry factory {}", factoryName);
+ Class clazz = ClassLoaderUtil
+ .loadClass(factoryName, org.apache.accumulo.core.metrics.MeterRegistryFactory.class);
+ org.apache.accumulo.core.metrics.MeterRegistryFactory factory =
+ clazz.getDeclaredConstructor().newInstance();
+ return factory.create();
+ } catch (ClassCastException ex) {
+ // empty. No valid metrics factory, fall through and then throw exception.
+ }
+ throw new ClassNotFoundException(
+ "Could not find appropriate class implementing a MetricsFactory for: " + factoryName);
+ }
+
+ @Override
+ public synchronized void close() {
+ LOG.info("Closing metrics registry");
+ if (jvmGcMetrics != null) {
+ jvmGcMetrics.close();
+ jvmGcMetrics = null;
+ }
+ if (composite != null) {
+ composite.close();
+ composite = null;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "MetricsCommonTags{tags=" + getCommonTags() + '}';
+ }
+}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
index 9f379909264..642d81a5c88 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
@@ -42,6 +42,7 @@
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.PropertyType;
import org.apache.accumulo.core.conf.PropertyType.PortRange;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.rpc.SslConnectionParams;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.UGIAssumingTransportFactory;
@@ -173,7 +174,7 @@ public static ServerAddress startServer(ServerContext context, String hostname,
// create the TimedProcessor outside the port search loop so we don't try to
// register the same
// metrics mbean more than once
- TimedProcessor timedProcessor = new TimedProcessor(processor);
+ TimedProcessor timedProcessor = new TimedProcessor(processor, context.getMetricsInfo());
HostAndPort[] addresses = getHostAndPorts(hostname, portHint);
try {
@@ -568,16 +569,16 @@ public static ServerAddress startTServer(final AccumuloConfiguration conf,
ThriftServerType serverType, TProcessor processor, String serverName, String threadName,
int numThreads, long threadTimeOut, long timeBetweenThreadChecks, long maxMessageSize,
SslConnectionParams sslParams, SaslServerConnectionParams saslParams,
- long serverSocketTimeout, int backlog, HostAndPort... addresses) {
+ long serverSocketTimeout, int backlog, MetricsInfo metricsInfo, HostAndPort... addresses) {
if (serverType == ThriftServerType.SASL) {
processor = updateSaslProcessor(serverType, processor);
}
try {
- return startTServer(serverType, new TimedProcessor(processor), serverName, threadName,
- numThreads, threadTimeOut, conf, timeBetweenThreadChecks, maxMessageSize, sslParams,
- saslParams, serverSocketTimeout, backlog, addresses);
+ return startTServer(serverType, new TimedProcessor(processor, metricsInfo), serverName,
+ threadName, numThreads, threadTimeOut, conf, timeBetweenThreadChecks, maxMessageSize,
+ sslParams, saslParams, serverSocketTimeout, backlog, addresses);
} catch (TTransportException e) {
throw new IllegalStateException(e);
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/rpc/TimedProcessor.java b/server/base/src/main/java/org/apache/accumulo/server/rpc/TimedProcessor.java
index 0307c42c078..30b7eec2e1f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/rpc/TimedProcessor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/rpc/TimedProcessor.java
@@ -20,7 +20,7 @@
import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import org.apache.accumulo.core.metrics.MetricsUtil;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.server.metrics.ThriftMetrics;
import org.apache.thrift.TException;
import org.apache.thrift.TProcessor;
@@ -35,10 +35,10 @@ public class TimedProcessor implements TProcessor {
private final ThriftMetrics thriftMetrics;
private long idleStart = 0;
- public TimedProcessor(TProcessor next) {
+ public TimedProcessor(final TProcessor next, final MetricsInfo metricsInfo) {
this.other = next;
thriftMetrics = new ThriftMetrics();
- MetricsUtil.initializeProducers(thriftMetrics);
+ metricsInfo.addMetricsProducers(thriftMetrics);
idleStart = System.nanoTime();
}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImplTest.java b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImplTest.java
index 1121b2b726c..3789bb3f419 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImplTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImplTest.java
@@ -35,7 +35,6 @@
import org.apache.accumulo.core.data.InstanceId;
import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.codec.VersionedProperties;
import org.apache.accumulo.server.conf.store.TablePropKey;
@@ -67,9 +66,6 @@ public void init() {
ticker = new TestTicker();
instanceId = InstanceId.of(UUID.randomUUID());
- PropStoreMetrics cacheMetrics = new PropStoreMetrics();
- MetricsUtil.initializeProducers(cacheMetrics);
-
tablePropKey =
TablePropKey.of(instanceId, TableId.of("t" + ThreadLocalRandom.current().nextInt(1, 1000)));
@@ -85,7 +81,7 @@ public void init() {
expect(context.getInstanceID()).andReturn(instanceId).anyTimes();
- cache = new PropCacheCaffeineImpl.Builder(zooPropLoader, cacheMetrics).forTests(ticker).build();
+ cache = new PropCacheCaffeineImpl.Builder(zooPropLoader).forTests(ticker).build();
}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropStoreEventTest.java b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropStoreEventTest.java
index 748013cd950..71247b4d3b5 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropStoreEventTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropStoreEventTest.java
@@ -39,7 +39,6 @@
import org.apache.accumulo.core.data.InstanceId;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.codec.VersionedPropCodec;
import org.apache.accumulo.server.conf.codec.VersionedProperties;
@@ -255,12 +254,9 @@ public void validateWatcherSetTest() throws Exception {
replay(context, zrw, readyMonitor);
- PropStoreMetrics metrics = new PropStoreMetrics();
- MetricsUtil.initializeProducers(metrics);
+ ZooPropLoader loader = new ZooPropLoader(zrw, propCodec, watcher);
- ZooPropLoader loader = new ZooPropLoader(zrw, propCodec, watcher, metrics);
-
- PropCacheCaffeineImpl cache = new PropCacheCaffeineImpl.Builder(loader, metrics).build();
+ PropCacheCaffeineImpl cache = new PropCacheCaffeineImpl.Builder(loader).build();
// load cache
var read1 = cache.get(tablePropKey);
diff --git a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/ZooPropLoaderTest.java b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/ZooPropLoaderTest.java
index d9f0f4020ef..c47fc49752c 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/ZooPropLoaderTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/ZooPropLoaderTest.java
@@ -23,7 +23,6 @@
import static org.apache.accumulo.core.conf.Property.TSERV_CLIENTPORT;
import static org.apache.accumulo.core.conf.Property.TSERV_NATIVEMAP_ENABLED;
import static org.apache.accumulo.core.conf.Property.TSERV_SCAN_MAX_OPENFILES;
-import static org.easymock.EasyMock.anyLong;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.capture;
import static org.easymock.EasyMock.createMock;
@@ -58,13 +57,9 @@
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class ZooPropLoaderTest {
- private static final Logger log = LoggerFactory.getLogger(ZooPropLoaderTest.class);
-
private PropCacheCaffeineImplTest.TestTicker ticker;
private InstanceId instanceId;
private ServerContext context;
@@ -72,7 +67,6 @@ public class ZooPropLoaderTest {
private VersionedPropCodec propCodec;
// mocks
- private PropStoreMetrics cacheMetrics;
private PropStoreWatcher propStoreWatcher;
private ZooReaderWriter zrw;
@@ -92,18 +86,16 @@ public void initCommonMocks() {
zrw = createMock(ZooReaderWriter.class);
- cacheMetrics = createMock(PropStoreMetrics.class);
-
propStoreWatcher = createMock(PropStoreWatcher.class);
// loader used in tests
- loader = new ZooPropLoader(zrw, propCodec, propStoreWatcher, cacheMetrics);
+ loader = new ZooPropLoader(zrw, propCodec, propStoreWatcher);
}
@AfterEach
public void verifyCommonMocks() {
- verify(context, zrw, propStoreWatcher, cacheMetrics);
+ verify(context, zrw, propStoreWatcher);
}
@Test
@@ -122,10 +114,7 @@ public void loadTest() throws Exception {
return (bytes);
}).once();
- cacheMetrics.addLoadTime(anyLong());
- expectLastCall().times(1);
-
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
assertNotNull(loader.load(propStoreKey));
}
@@ -158,13 +147,10 @@ public void loadAndCacheTest() throws Exception {
return (bytes);
}).once();
- cacheMetrics.addLoadTime(anyLong());
- expectLastCall().times(1);
-
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
PropCacheCaffeineImpl cache =
- new PropCacheCaffeineImpl.Builder(loader, cacheMetrics).forTests(ticker).build();
+ new PropCacheCaffeineImpl.Builder(loader).forTests(ticker).build();
// load into cache
assertNotNull(cache.get(propStoreKey));
@@ -189,17 +175,12 @@ public void loadFailTest() throws Exception {
propStoreWatcher.signalZkChangeEvent(eq(propStoreKey));
expectLastCall();
- cacheMetrics.incrZkError();
- expectLastCall().once();
-
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
PropCacheCaffeineImpl cache =
- new PropCacheCaffeineImpl.Builder(loader, cacheMetrics).forTests(ticker).build();
+ new PropCacheCaffeineImpl.Builder(loader).forTests(ticker).build();
assertNull(cache.get(propStoreKey));
-
- log.debug("Metrics: {}", cacheMetrics);
}
/**
@@ -225,16 +206,10 @@ public void expireTest() throws Exception {
return (bytes);
}).times(2);
- cacheMetrics.addLoadTime(anyLong());
- expectLastCall().times(2);
-
- cacheMetrics.incrEviction();
- expectLastCall().once();
-
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
PropCacheCaffeineImpl cache =
- new PropCacheCaffeineImpl.Builder(loader, cacheMetrics).forTests(ticker).build();
+ new PropCacheCaffeineImpl.Builder(loader).forTests(ticker).build();
// load cache
assertNotNull(cache.get(propStoreKey));
@@ -278,17 +253,10 @@ public void reloadExceptionTest() throws Exception {
propStoreWatcher.signalCacheChangeEvent(anyObject());
expectLastCall().anyTimes();
- cacheMetrics.addLoadTime(anyLong());
- expectLastCall().times(1);
- cacheMetrics.incrEviction();
- expectLastCall().times(1);
- cacheMetrics.incrZkError();
- expectLastCall().times(1);
-
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
PropCacheCaffeineImpl cache =
- new PropCacheCaffeineImpl.Builder(loader, cacheMetrics).forTests(ticker).build();
+ new PropCacheCaffeineImpl.Builder(loader).forTests(ticker).build();
// prime cache
assertNotNull(cache.get(propStoreKey));
@@ -307,10 +275,10 @@ public void reloadExceptionTest() throws Exception {
@Test
public void getIfCachedTest() {
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
PropCacheCaffeineImpl cache =
- new PropCacheCaffeineImpl.Builder(loader, cacheMetrics).forTests(ticker).build();
+ new PropCacheCaffeineImpl.Builder(loader).forTests(ticker).build();
assertNull(cache.getIfCached(propStoreKey));
@@ -348,13 +316,10 @@ public void removeTest() throws Exception {
return (bytes);
}).once();
- cacheMetrics.addLoadTime(anyLong());
- expectLastCall().times(2);
-
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
PropCacheCaffeineImpl cache =
- new PropCacheCaffeineImpl.Builder(loader, cacheMetrics).forTests(ticker).build();
+ new PropCacheCaffeineImpl.Builder(loader).forTests(ticker).build();
// load into cache
assertNotNull(cache.get(sysPropKey));
@@ -399,13 +364,10 @@ public void removeAllTest() throws Exception {
return (bytes);
}).once();
- cacheMetrics.addLoadTime(anyLong());
- expectLastCall().times(2);
-
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
PropCacheCaffeineImpl cache =
- new PropCacheCaffeineImpl.Builder(loader, cacheMetrics).forTests(ticker).build();
+ new PropCacheCaffeineImpl.Builder(loader).forTests(ticker).build();
// load into cache
assertNotNull(cache.get(sysPropKey));
@@ -420,10 +382,10 @@ public void removeAllTest() throws Exception {
@Test
public void getIfCachedNotPresentTest() {
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
PropCacheCaffeineImpl cache =
- new PropCacheCaffeineImpl.Builder(loader, cacheMetrics).forTests(ticker).build();
+ new PropCacheCaffeineImpl.Builder(loader).forTests(ticker).build();
// load into cache
assertNull(cache.getIfCached(propStoreKey));
@@ -451,7 +413,7 @@ public void captureExampleTest() throws Exception {
return propCodec.toBytes(vProps);
}).anyTimes();
- replay(context, zrw, propStoreWatcher, cacheMetrics);
+ replay(context, zrw, propStoreWatcher);
Stat statCheck = new Stat();
statCheck.setVersion(9);
diff --git a/server/base/src/test/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImplTest.java b/server/base/src/test/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImplTest.java
new file mode 100644
index 00000000000..ff0f958dd57
--- /dev/null
+++ b/server/base/src/test/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImplTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.util.Map;
+
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.server.ServerContext;
+import org.easymock.EasyMock;
+import org.junit.jupiter.api.Test;
+
+public class MeterRegistryEnvPropImplTest {
+
+ @Test
+ public void customParamsTest() {
+ ServerContext context = EasyMock.createMock(ServerContext.class);
+ ConfigurationCopy conf = new ConfigurationCopy();
+ conf.set("unknown", "none");
+ conf.set("general.custom.metrics.opts.sample.p1", "v1");
+ conf.set("general.custom.metrics.opts.sample.p2", "v2");
+
+ expect(context.getConfiguration()).andReturn(conf).anyTimes();
+
+ replay(context);
+ MeterRegistryEnvPropImpl env = new MeterRegistryEnvPropImpl(context);
+ assertEquals(Map.of("sample.p1", "v1", "sample.p2", "v2"), env.getOptions());
+ assertNotNull(env.getServiceEnv());
+ }
+}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/metrics/MetricsInfoImplTest.java b/server/base/src/test/java/org/apache/accumulo/server/metrics/MetricsInfoImplTest.java
new file mode 100644
index 00000000000..5ffb6156858
--- /dev/null
+++ b/server/base/src/test/java/org/apache/accumulo/server/metrics/MetricsInfoImplTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.metrics;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.mock;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.util.Map;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.server.ServerContext;
+import org.junit.jupiter.api.Test;
+
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+
+public class MetricsInfoImplTest {
+ @Test
+ public void factoryTest() throws Exception {
+
+ ServerContext context = mock(ServerContext.class);
+ AccumuloConfiguration conf = mock(AccumuloConfiguration.class);
+ expect(context.getConfiguration()).andReturn(conf).anyTimes();
+ expect(conf.getAllPropertiesWithPrefixStripped(anyObject())).andReturn(Map.of()).anyTimes();
+ expect(conf.newDeriver(anyObject())).andReturn(Map::of).anyTimes();
+ replay(context, conf);
+ assertNotNull(MetricsInfoImpl.getRegistryFromFactory(SPIFactory.class.getName(), context));
+
+ assertNotNull(
+ MetricsInfoImpl.getRegistryFromFactory(DeprecatedFactory.class.getName(), context));
+
+ assertThrows(ClassNotFoundException.class,
+ () -> MetricsInfoImpl.getRegistryFromFactory(String.class.getName(), context));
+
+ verify(context, conf);
+ }
+
+ @SuppressWarnings({"deprecation",
+ "support for org.apache.accumulo.core.metrics.MeterRegistryFactory can be removed in 3.1"})
+ static final class DeprecatedFactory
+ implements org.apache.accumulo.core.metrics.MeterRegistryFactory {
+ DeprecatedFactory() {
+
+ }
+
+ @Override
+ public MeterRegistry create() {
+ return new SimpleMeterRegistry();
+ }
+ }
+
+ static class SPIFactory implements org.apache.accumulo.core.spi.metrics.MeterRegistryFactory {
+
+ SPIFactory() {
+
+ }
+
+ @Override
+ public MeterRegistry create(final InitParameters params) {
+ return new SimpleMeterRegistry();
+ }
+ }
+}
diff --git a/server/base/src/test/java/org/apache/accumulo/server/rpc/TServerUtilsTest.java b/server/base/src/test/java/org/apache/accumulo/server/rpc/TServerUtilsTest.java
index 40b46aee668..d8230e80d55 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/rpc/TServerUtilsTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/rpc/TServerUtilsTest.java
@@ -18,8 +18,10 @@
*/
package org.apache.accumulo.server.rpc;
+import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -41,6 +43,8 @@
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.InstanceId;
+import org.apache.accumulo.core.metrics.MetricsInfo;
+import org.apache.accumulo.core.metrics.MetricsProducer;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.client.ClientServiceHandler;
@@ -54,6 +58,7 @@
public class TServerUtilsTest {
private ServerContext context;
+ private MetricsInfo metricsInfo;
private final ConfigurationCopy conf = new ConfigurationCopy(DefaultConfiguration.getInstance());
@BeforeEach
@@ -72,12 +77,16 @@ public void createMockServerContext() {
expect(context.getSaslParams()).andReturn(null).anyTimes();
expect(context.getClientTimeoutInMillis()).andReturn((long) 1000).anyTimes();
expect(context.getSecurityOperation()).andReturn(null).anyTimes();
- replay(context);
+ metricsInfo = createMock(MetricsInfo.class);
+ metricsInfo.addMetricsProducers(anyObject(MetricsProducer.class));
+ expectLastCall().anyTimes();
+ expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
+ replay(context, metricsInfo);
}
@AfterEach
public void verifyMockServerContext() {
- verify(context);
+ verify(context, metricsInfo);
}
@Test
@@ -206,7 +215,7 @@ public void testStartServerNonDefaultPorts() throws Exception {
assertNotNull(server);
// Finally ensure that the TServer is using the last port (i.e. port search worked)
- assertTrue(address.getAddress().getPort() == tserverFinalPort);
+ assertEquals(address.getAddress().getPort(), tserverFinalPort);
} finally {
if (server != null) {
server.stop();
@@ -259,9 +268,9 @@ public void testStartServerPortRangeFirstPortUsed() throws Exception {
}
private int[] findTwoFreeSequentialPorts(int startingAddress) throws UnknownHostException {
- boolean sequential = false;
+ boolean sequential;
int low = startingAddress;
- int high = 0;
+ int high;
do {
low = getFreePort(low);
high = getFreePort(low + 1);
diff --git a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 75690df777a..2b0dee53b7e 100644
--- a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++ b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -21,7 +21,6 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly;
-import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
import java.util.HashSet;
import java.util.List;
@@ -61,7 +60,7 @@
import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
-import org.apache.accumulo.core.metrics.MetricsUtil;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
@@ -265,14 +264,9 @@ public void run() {
throw new IllegalStateException("Exception getting Coordinator lock", e);
}
- try {
- MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName,
- clientAddress, getContext().getInstanceName());
- } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
- | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
- | SecurityException e1) {
- LOG.error("Error initializing metrics, metrics will not be emitted.", e1);
- }
+ MetricsInfo metricsInfo = getContext().getMetricsInfo();
+ metricsInfo.addServiceTags(getApplicationName(), clientAddress);
+ metricsInfo.init();
// On a re-start of the coordinator it's possible that external compactions are in-progress.
// Attempt to get the running compactions on the compactors and then resolve which tserver
diff --git a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
index 87e7471bef3..1a97e3df927 100644
--- a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
+++ b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
@@ -46,6 +46,7 @@
import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.tabletserver.thrift.TCompactionQueueSummary;
@@ -227,6 +228,8 @@ public void testCoordinatorColdStartNoCompactions() throws Exception {
ServerContext context = PowerMock.createNiceMock(ServerContext.class);
expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
+ MetricsInfo metricsInfo = PowerMock.createNiceMock(MetricsInfo.class);
+ expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
PowerMock.mockStatic(ExternalCompactionUtil.class);
List runningCompactions = new ArrayList<>();
@@ -278,6 +281,8 @@ public void testCoordinatorColdStart() throws Exception {
ServerContext context = PowerMock.createNiceMock(ServerContext.class);
expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
+ MetricsInfo metricsInfo = PowerMock.createNiceMock(MetricsInfo.class);
+ expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
expect(context.rpcCreds()).andReturn(creds);
@@ -350,6 +355,8 @@ public void testCoordinatorRestartNoRunningCompactions() throws Exception {
ServerContext context = PowerMock.createNiceMock(ServerContext.class);
expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
+ MetricsInfo metricsInfo = PowerMock.createNiceMock(MetricsInfo.class);
+ expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
expect(context.rpcCreds()).andReturn(creds);
@@ -425,6 +432,8 @@ public void testCoordinatorRestartOneRunningCompaction() throws Exception {
ServerContext context = PowerMock.createNiceMock(ServerContext.class);
expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
+ MetricsInfo metricsInfo = PowerMock.createNiceMock(MetricsInfo.class);
+ expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
expect(context.rpcCreds()).andReturn(creds);
@@ -505,6 +514,8 @@ public void testGetCompactionJob() throws Exception {
ServerContext context = PowerMock.createNiceMock(ServerContext.class);
expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
+ MetricsInfo metricsInfo = PowerMock.createNiceMock(MetricsInfo.class);
+ expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
expect(context.rpcCreds()).andReturn(creds).anyTimes();
diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 5e09e48fdf1..04e33a9c88f 100644
--- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -23,7 +23,6 @@
import static org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly;
import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
import java.security.SecureRandom;
import java.util.ArrayList;
@@ -73,8 +72,8 @@
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.metrics.MetricsProducer;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
@@ -593,15 +592,11 @@ public void run() {
throw new RuntimeException("Error registering compactor in ZooKeeper", e);
}
- try {
- MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName,
- clientAddress, getContext().getInstanceName());
- } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
- | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
- | SecurityException e1) {
- LOG.error("Error initializing metrics, metrics will not be emitted.", e1);
- }
- MetricsUtil.initializeProducers(this);
+ MetricsInfo metricsInfo = getContext().getMetricsInfo();
+ metricsInfo.addServiceTags(getApplicationName(), clientAddress);
+
+ metricsInfo.addMetricsProducers(this);
+ metricsInfo.init();
var watcher = new CompactionWatcher(getConfiguration());
var schedExecutor = ThreadPools.getServerThreadPools()
diff --git a/server/compactor/src/test/java/org/apache/accumulo/compactor/CompactorTest.java b/server/compactor/src/test/java/org/apache/accumulo/compactor/CompactorTest.java
index f7e979820e8..c747d9392c8 100644
--- a/server/compactor/src/test/java/org/apache/accumulo/compactor/CompactorTest.java
+++ b/server/compactor/src/test/java/org/apache/accumulo/compactor/CompactorTest.java
@@ -45,6 +45,7 @@
import org.apache.accumulo.core.fate.zookeeper.ZooCache;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.tabletserver.thrift.TCompactionStats;
import org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob;
import org.apache.accumulo.core.util.Halt;
@@ -323,6 +324,9 @@ public void testCompactionSucceeds() throws Exception {
ServerContext context = PowerMock.createNiceMock(ServerContext.class);
expect(context.getConfiguration()).andReturn(conf).anyTimes();
+ MetricsInfo metricsInfo = PowerMock.createNiceMock(MetricsInfo.class);
+ expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
+
ZooReaderWriter zrw = PowerMock.createNiceMock(ZooReaderWriter.class);
ZooKeeper zk = PowerMock.createNiceMock(ZooKeeper.class);
expect(context.getZooReaderWriter()).andReturn(zrw).anyTimes();
@@ -370,6 +374,9 @@ public void testCompactionFails() throws Exception {
ServerContext context = PowerMock.createNiceMock(ServerContext.class);
expect(context.getConfiguration()).andReturn(conf).anyTimes();
+ MetricsInfo metricsInfo = PowerMock.createNiceMock(MetricsInfo.class);
+ expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
+
ZooReaderWriter zrw = PowerMock.createNiceMock(ZooReaderWriter.class);
ZooKeeper zk = PowerMock.createNiceMock(ZooKeeper.class);
expect(context.getZooReaderWriter()).andReturn(zrw).anyTimes();
@@ -418,6 +425,9 @@ public void testCompactionInterrupted() throws Exception {
ServerContext context = PowerMock.createNiceMock(ServerContext.class);
expect(context.getConfiguration()).andReturn(conf).anyTimes();
+ MetricsInfo metricsInfo = PowerMock.createNiceMock(MetricsInfo.class);
+ expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
+
ZooReaderWriter zrw = PowerMock.createNiceMock(ZooReaderWriter.class);
ZooKeeper zk = PowerMock.createNiceMock(ZooKeeper.class);
expect(context.getZooReaderWriter()).andReturn(zrw).anyTimes();
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 6e186e21579..6a7025a979e 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -23,7 +23,6 @@
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;
@@ -41,7 +40,7 @@
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
-import org.apache.accumulo.core.metrics.MetricsUtil;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.trace.thrift.TInfo;
@@ -167,16 +166,11 @@ public void run() {
System.exit(1);
}
- try {
- MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, address,
- getContext().getInstanceName());
- MetricsUtil.initializeProducers(new GcMetrics(this));
- } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
- | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
- | SecurityException e1) {
- log.error("Error initializing metrics, metrics will not be emitted.", e1);
- }
+ MetricsInfo metricsInfo = getContext().getMetricsInfo();
+ metricsInfo.addServiceTags(getApplicationName(), address);
+ metricsInfo.addMetricsProducers(new GcMetrics(this));
+ metricsInfo.init();
try {
long delay = getStartDelay();
log.debug("Sleeping for {} milliseconds before beginning garbage collection cycles", delay);
@@ -407,7 +401,8 @@ private HostAndPort startStatsService() {
getContext().getThriftServerType(), processor, this.getClass().getSimpleName(),
"GC Monitor Service", 2, ThreadPools.DEFAULT_TIMEOUT_MILLISECS, 1000, maxMessageSize,
getContext().getServerSslParams(), getContext().getSaslParams(), 0,
- getConfiguration().getCount(Property.RPC_BACKLOG), addresses);
+ getConfiguration().getCount(Property.RPC_BACKLOG), getContext().getMetricsInfo(),
+ addresses);
log.debug("Starting garbage collector listening on " + server.address);
return server.address;
}
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 3467bea76d0..86a1dd71d3d 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@ -28,7 +28,6 @@
import static org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly;
import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
@@ -97,7 +96,8 @@
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
-import org.apache.accumulo.core.metrics.MetricsUtil;
+import org.apache.accumulo.core.metrics.MetricsInfo;
+import org.apache.accumulo.core.metrics.MetricsProducer;
import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.spi.balancer.BalancerEnvironment;
@@ -1100,15 +1100,12 @@ public void run() {
managerUpgrading.set(true);
}
- try {
- MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName,
- sa.getAddress(), getContext().getInstanceName());
- ManagerMetrics.init(getConfiguration(), this);
- } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
- | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
- | SecurityException e1) {
- log.error("Error initializing metrics, metrics will not be emitted.", e1);
- }
+ MetricsInfo metricsInfo = getContext().getMetricsInfo();
+ metricsInfo.addServiceTags(getApplicationName(), sa.getAddress());
+
+ var producers = ManagerMetrics.getProducers(getConfiguration(), this);
+ metricsInfo.addMetricsProducers(producers.toArray(new MetricsProducer[0]));
+ metricsInfo.init();
recoveryManager = new RecoveryManager(this, timeToCacheRecoveryWalExistence);
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java
index 680b13558a8..f0502bce988 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java
@@ -18,11 +18,12 @@
*/
package org.apache.accumulo.manager.metrics;
-import static java.util.Objects.requireNonNull;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.metrics.MetricsUtil;
+import org.apache.accumulo.core.metrics.MetricsProducer;
import org.apache.accumulo.manager.Manager;
import org.apache.accumulo.manager.metrics.fate.FateMetrics;
import org.slf4j.Logger;
@@ -32,15 +33,13 @@ public class ManagerMetrics {
private final static Logger log = LoggerFactory.getLogger(ManagerMetrics.class);
- public static void init(AccumuloConfiguration conf, Manager m) {
- requireNonNull(conf, "AccumuloConfiguration must not be null");
+ public static List getProducers(AccumuloConfiguration conf, Manager m) {
+ ArrayList producers = new ArrayList<>();
@SuppressWarnings("deprecation")
ReplicationMetrics replMetrics = new ReplicationMetrics(m);
- MetricsUtil.initializeProducers(replMetrics);
- log.info("Registered replication metrics module");
- MetricsUtil.initializeProducers(new FateMetrics(m.getContext(),
+ producers.add(replMetrics);
+ producers.add(new FateMetrics(m.getContext(),
conf.getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL)));
- log.info("Registered FATE metrics module");
+ return producers;
}
-
}
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ReplicationMetrics.java b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ReplicationMetrics.java
index a5e9213948c..931fb302152 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ReplicationMetrics.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ReplicationMetrics.java
@@ -32,7 +32,6 @@
import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.metrics.MetricsProducer;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.replication.ReplicationTable;
import org.apache.accumulo.core.replication.ReplicationTarget;
import org.apache.accumulo.core.util.threads.ThreadPools;
@@ -154,13 +153,10 @@ protected void addReplicationQueueTimeMetrics() {
@Override
public void registerMetrics(MeterRegistry registry) {
- replicationQueueTimer = registry.timer(METRICS_REPLICATION_QUEUE, MetricsUtil.getCommonTags());
- pendingFiles = registry.gauge(METRICS_REPLICATION_PENDING_FILES, MetricsUtil.getCommonTags(),
- new AtomicLong(0));
- numPeers = registry.gauge(METRICS_REPLICATION_PEERS, MetricsUtil.getCommonTags(),
- new AtomicInteger(0));
- maxReplicationThreads = registry.gauge(METRICS_REPLICATION_THREADS, MetricsUtil.getCommonTags(),
- new AtomicInteger(0));
+ replicationQueueTimer = registry.timer(METRICS_REPLICATION_QUEUE);
+ pendingFiles = registry.gauge(METRICS_REPLICATION_PENDING_FILES, new AtomicLong(0));
+ numPeers = registry.gauge(METRICS_REPLICATION_PEERS, new AtomicInteger(0));
+ maxReplicationThreads = registry.gauge(METRICS_REPLICATION_THREADS, new AtomicInteger(0));
ScheduledExecutorService scheduler = ThreadPools.getServerThreadPools()
.createScheduledExecutorService(1, "replicationMetricsPoller");
diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
index 800455fe0e8..bed2e534115 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.manager.metrics.fate;
+import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
@@ -28,7 +29,6 @@
import org.apache.accumulo.core.fate.ReadOnlyTStore;
import org.apache.accumulo.core.fate.ZooStore;
import org.apache.accumulo.core.metrics.MetricsProducer;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.server.ServerContext;
import org.apache.zookeeper.KeeperException;
@@ -37,6 +37,7 @@
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
public class FateMetrics implements MetricsProducer {
@@ -127,27 +128,31 @@ private void update() {
@Override
public void registerMetrics(final MeterRegistry registry) {
- totalCurrentOpsGauge = registry.gauge(METRICS_FATE_TOTAL_IN_PROGRESS,
- MetricsUtil.getCommonTags(), new AtomicLong(0));
- totalOpsGauge =
- registry.gauge(METRICS_FATE_OPS_ACTIVITY, MetricsUtil.getCommonTags(), new AtomicLong(0));
- fateErrorsGauge = registry.gauge(METRICS_FATE_ERRORS,
- Tags.concat(MetricsUtil.getCommonTags(), "type", "zk.connection"), new AtomicLong(0));
- newTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(), "state",
- ReadOnlyTStore.TStatus.NEW.name().toLowerCase()), new AtomicLong(0));
- submittedTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
- "state", ReadOnlyTStore.TStatus.SUBMITTED.name().toLowerCase()), new AtomicLong(0));
- inProgressTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
- "state", ReadOnlyTStore.TStatus.IN_PROGRESS.name().toLowerCase()), new AtomicLong(0));
- failedInProgressTxGauge =
- registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(), "state",
- ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS.name().toLowerCase()), new AtomicLong(0));
- failedTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
- "state", ReadOnlyTStore.TStatus.FAILED.name().toLowerCase()), new AtomicLong(0));
- successfulTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
- "state", ReadOnlyTStore.TStatus.SUCCESSFUL.name().toLowerCase()), new AtomicLong(0));
- unknownTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
- "state", ReadOnlyTStore.TStatus.UNKNOWN.name().toLowerCase()), new AtomicLong(0));
+ totalCurrentOpsGauge = registry.gauge(METRICS_FATE_TOTAL_IN_PROGRESS, new AtomicLong(0));
+ totalOpsGauge = registry.gauge(METRICS_FATE_OPS_ACTIVITY, new AtomicLong(0));
+ fateErrorsGauge = registry.gauge(METRICS_FATE_ERRORS, List.of(Tag.of("type", "zk.connection")),
+ new AtomicLong(0));
+ newTxGauge = registry.gauge(METRICS_FATE_TX,
+ List.of(Tag.of("state", ReadOnlyTStore.TStatus.NEW.name().toLowerCase())),
+ new AtomicLong(0));
+ submittedTxGauge = registry.gauge(METRICS_FATE_TX,
+ List.of(Tag.of("state", ReadOnlyTStore.TStatus.SUBMITTED.name().toLowerCase())),
+ new AtomicLong(0));
+ inProgressTxGauge = registry.gauge(METRICS_FATE_TX,
+ List.of(Tag.of("state", ReadOnlyTStore.TStatus.IN_PROGRESS.name().toLowerCase())),
+ new AtomicLong(0));
+ failedInProgressTxGauge = registry.gauge(METRICS_FATE_TX,
+ List.of(Tag.of("state", ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS.name().toLowerCase())),
+ new AtomicLong(0));
+ failedTxGauge = registry.gauge(METRICS_FATE_TX,
+ List.of(Tag.of("state", ReadOnlyTStore.TStatus.FAILED.name().toLowerCase())),
+ new AtomicLong(0));
+ successfulTxGauge = registry.gauge(METRICS_FATE_TX,
+ List.of(Tag.of("state", ReadOnlyTStore.TStatus.SUCCESSFUL.name().toLowerCase())),
+ new AtomicLong(0));
+ unknownTxGauge = registry.gauge(METRICS_FATE_TX,
+ List.of(Tag.of("state", ReadOnlyTStore.TStatus.UNKNOWN.name().toLowerCase())),
+ new AtomicLong(0));
update();
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 03ba9a34456..96de9049e59 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -62,6 +62,7 @@
import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo;
import org.apache.accumulo.core.master.thrift.TableInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
@@ -485,6 +486,11 @@ public void run() {
}
log.debug("Using {} to advertise monitor location in ZooKeeper", advertiseHost);
+ MetricsInfo metricsInfo = getContext().getMetricsInfo();
+ metricsInfo.addServiceTags(getApplicationName(),
+ HostAndPort.fromParts(advertiseHost, livePort));
+ metricsInfo.init();
+
try {
URL url = new URL(server.isSecure() ? "https" : "http", advertiseHost, server.getPort(), "/");
final String path = context.getZooKeeperRoot() + Constants.ZMONITOR_HTTP_ADDR;
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index 5559fb148b7..bcba26aa135 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@ -23,7 +23,6 @@
import java.io.IOException;
import java.io.UncheckedIOException;
-import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -71,7 +70,7 @@
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
-import org.apache.accumulo.core.metrics.MetricsUtil;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.spi.scan.ScanServerSelector;
import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
@@ -370,17 +369,13 @@ public void run() {
throw new RuntimeException("Failed to start the compactor client service", e1);
}
- try {
- MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName,
- clientAddress, getContext().getInstanceName());
- } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
- | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
- | SecurityException e1) {
- LOG.error("Error initializing metrics, metrics will not be emitted.", e1);
- }
+ MetricsInfo metricsInfo = getContext().getMetricsInfo();
+ metricsInfo.addServiceTags(getApplicationName(), clientAddress);
+
scanMetrics = new TabletServerScanMetrics();
- MetricsUtil.initializeProducers(scanMetrics);
+ metricsInfo.addMetricsProducers(scanMetrics);
+ metricsInfo.init();
// We need to set the compaction manager so that we don't get an NPE in CompactableImpl.close
ServiceLock lock = announceExistence();
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 8e9013fdc5f..b24c7cf30db 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -30,7 +30,6 @@
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
import java.security.SecureRandom;
import java.time.Duration;
@@ -89,7 +88,7 @@
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
-import org.apache.accumulo.core.metrics.MetricsUtil;
+import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment;
@@ -760,21 +759,17 @@ public void run() {
throw new RuntimeException("Failed to start the tablet client service", e1);
}
- try {
- MetricsUtil.initializeMetrics(context.getConfiguration(), this.applicationName, clientAddress,
- getContext().getInstanceName());
- } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
- | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
- | SecurityException e1) {
- log.error("Error initializing metrics, metrics will not be emitted.", e1);
- }
+ MetricsInfo metricsInfo = getContext().getMetricsInfo();
+ metricsInfo.addServiceTags(getApplicationName(), clientAddress);
metrics = new TabletServerMetrics(this);
updateMetrics = new TabletServerUpdateMetrics();
scanMetrics = new TabletServerScanMetrics();
mincMetrics = new TabletServerMinCMetrics();
ceMetrics = new CompactionExecutorsMetrics();
- MetricsUtil.initializeProducers(metrics, updateMetrics, scanMetrics, mincMetrics, ceMetrics);
+
+ metricsInfo.addMetricsProducers(metrics, updateMetrics, scanMetrics, mincMetrics, ceMetrics);
+ metricsInfo.init();
this.compactionManager = new CompactionManager(() -> Iterators
.transform(onlineTablets.snapshot().values().iterator(), Tablet::asCompactable),
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index 3d80f9c1ed0..c3ad1fd830f 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@ -140,7 +140,7 @@ private void modifyThreadPoolSizesAtRuntime(IntSupplier maxThreads, String name,
}
private ThreadPoolExecutor createPriorityExecutor(ScanExecutorConfig sec,
- Map> scanExecQueues) {
+ Map> scanExecQueues, final boolean enableMetrics) {
BlockingQueue queue;
@@ -188,7 +188,7 @@ public ServiceEnvironment getServiceEnv() {
ThreadPoolExecutor es = ThreadPools.getServerThreadPools().getPoolBuilder("scan-" + sec.name)
.numCoreThreads(sec.getCurrentMaxThreads()).numMaxThreads(sec.getCurrentMaxThreads())
.withTimeOut(0L, MILLISECONDS).withQueue(queue).atPriority(sec.priority)
- .enableThreadPoolMetrics().build();
+ .enableThreadPoolMetrics(enableMetrics).build();
modifyThreadPoolSizesAtRuntime(sec::getCurrentMaxThreads, "scan-" + sec.name, es);
return es;
@@ -251,6 +251,7 @@ public Config getConfig() {
public TabletServerResourceManager(ServerContext context, TabletHostingServer tserver) {
this.context = context;
final AccumuloConfiguration acuConf = context.getConfiguration();
+ final boolean enableMetrics = context.getMetricsInfo().isMetricsEnabled(); // acuConf.getBoolean(Property.GENERAL_MICROMETER_ENABLED);
long maxMemory = acuConf.getAsBytes(Property.TSERV_MAXMEM);
boolean usingNativeMap = acuConf.getBoolean(Property.TSERV_NATIVEMAP_ENABLED);
if (usingNativeMap) {
@@ -307,20 +308,20 @@ public TabletServerResourceManager(ServerContext context, TabletHostingServer ts
() -> context.getConfiguration().getCount(Property.TSERV_MINC_MAXCONCURRENT),
"minor compactor", minorCompactionThreadPool);
- splitThreadPool =
- ThreadPools.getServerThreadPools().getPoolBuilder("splitter").numCoreThreads(0)
- .numMaxThreads(1).withTimeOut(1, SECONDS).enableThreadPoolMetrics().build();
+ splitThreadPool = ThreadPools.getServerThreadPools().getPoolBuilder("splitter")
+ .numCoreThreads(0).numMaxThreads(1).withTimeOut(1, SECONDS)
+ .enableThreadPoolMetrics(enableMetrics).build();
- defaultSplitThreadPool =
- ThreadPools.getServerThreadPools().getPoolBuilder("md splitter").numCoreThreads(0)
- .numMaxThreads(1).withTimeOut(60, SECONDS).enableThreadPoolMetrics().build();
+ defaultSplitThreadPool = ThreadPools.getServerThreadPools().getPoolBuilder("md splitter")
+ .numCoreThreads(0).numMaxThreads(1).withTimeOut(60, SECONDS)
+ .enableThreadPoolMetrics(enableMetrics).build();
defaultMigrationPool = ThreadPools.getServerThreadPools()
.getPoolBuilder("metadata tablet migration").numCoreThreads(0).numMaxThreads(1)
- .withTimeOut(60, SECONDS).enableThreadPoolMetrics().build();
+ .withTimeOut(60, SECONDS).enableThreadPoolMetrics(enableMetrics).build();
migrationPool = ThreadPools.getServerThreadPools().createExecutorService(acuConf,
- Property.TSERV_MIGRATE_MAXCONCURRENT, true);
+ Property.TSERV_MIGRATE_MAXCONCURRENT, enableMetrics);
modifyThreadPoolSizesAtRuntime(
() -> context.getConfiguration().getCount(Property.TSERV_MIGRATE_MAXCONCURRENT),
"tablet migration", migrationPool);
@@ -331,31 +332,31 @@ public TabletServerResourceManager(ServerContext context, TabletHostingServer ts
// individual tablet server run
// concurrent assignments would put more load on the metadata table at startup
assignmentPool = ThreadPools.getServerThreadPools().createExecutorService(acuConf,
- Property.TSERV_ASSIGNMENT_MAXCONCURRENT, true);
+ Property.TSERV_ASSIGNMENT_MAXCONCURRENT, enableMetrics);
modifyThreadPoolSizesAtRuntime(
() -> context.getConfiguration().getCount(Property.TSERV_ASSIGNMENT_MAXCONCURRENT),
"tablet assignment", assignmentPool);
assignMetaDataPool = ThreadPools.getServerThreadPools()
.getPoolBuilder("metadata tablet assignment").numCoreThreads(0).numMaxThreads(1)
- .withTimeOut(60, SECONDS).enableThreadPoolMetrics().build();
+ .withTimeOut(60, SECONDS).enableThreadPoolMetrics(enableMetrics).build();
activeAssignments = new ConcurrentHashMap<>();
summaryRetrievalPool = ThreadPools.getServerThreadPools().createExecutorService(acuConf,
- Property.TSERV_SUMMARY_RETRIEVAL_THREADS, true);
+ Property.TSERV_SUMMARY_RETRIEVAL_THREADS, enableMetrics);
modifyThreadPoolSizesAtRuntime(
() -> context.getConfiguration().getCount(Property.TSERV_SUMMARY_RETRIEVAL_THREADS),
"summary file retriever", summaryRetrievalPool);
summaryRemotePool = ThreadPools.getServerThreadPools().createExecutorService(acuConf,
- Property.TSERV_SUMMARY_REMOTE_THREADS, true);
+ Property.TSERV_SUMMARY_REMOTE_THREADS, enableMetrics);
modifyThreadPoolSizesAtRuntime(
() -> context.getConfiguration().getCount(Property.TSERV_SUMMARY_REMOTE_THREADS),
"summary remote", summaryRemotePool);
summaryPartitionPool = ThreadPools.getServerThreadPools().createExecutorService(acuConf,
- Property.TSERV_SUMMARY_PARTITION_THREADS, true);
+ Property.TSERV_SUMMARY_PARTITION_THREADS, enableMetrics);
modifyThreadPoolSizesAtRuntime(
() -> context.getConfiguration().getCount(Property.TSERV_SUMMARY_PARTITION_THREADS),
"summary partition", summaryPartitionPool);
@@ -364,8 +365,8 @@ public TabletServerResourceManager(ServerContext context, TabletHostingServer ts
Collection scanExecCfg = acuConf.getScanExecutors(isScanServer);
Map> scanExecQueues = new HashMap<>();
- scanExecutors = scanExecCfg.stream().collect(
- toUnmodifiableMap(cfg -> cfg.name, cfg -> createPriorityExecutor(cfg, scanExecQueues)));
+ scanExecutors = scanExecCfg.stream().collect(toUnmodifiableMap(cfg -> cfg.name,
+ cfg -> createPriorityExecutor(cfg, scanExecQueues, enableMetrics)));
scanExecutorChoices = scanExecCfg.stream().collect(toUnmodifiableMap(cfg -> cfg.name,
cfg -> new ScanExecutorImpl(cfg, scanExecQueues.get(cfg.name))));
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMinCMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMinCMetrics.java
index 3df8aff090c..a7f402343c7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMinCMetrics.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMinCMetrics.java
@@ -21,7 +21,6 @@
import java.time.Duration;
import org.apache.accumulo.core.metrics.MetricsProducer;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
@@ -42,11 +41,10 @@ public void addQueued(long value) {
@Override
public void registerMetrics(MeterRegistry registry) {
activeMinc = Timer.builder(METRICS_MINC_RUNNING).description("Minor compactions time active")
- .tags(MetricsUtil.getCommonTags()).register(registry);
+ .register(registry);
- queuedMinc =
- Timer.builder(METRICS_MINC_QUEUED).description("Queued minor compactions time queued")
- .tags(MetricsUtil.getCommonTags()).register(registry);
+ queuedMinc = Timer.builder(METRICS_MINC_QUEUED)
+ .description("Queued minor compactions time queued").register(registry);
}
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java
index d6bd26eb7f5..492a6a8c803 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java
@@ -23,7 +23,6 @@
import java.util.concurrent.atomic.LongAdder;
import org.apache.accumulo.core.metrics.MetricsProducer;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.DistributionSummary;
@@ -128,18 +127,14 @@ public void registerMetrics(MeterRegistry registry) {
.description("Results per scan").register(registry);
yields =
DistributionSummary.builder(METRICS_SCAN_YIELDS).description("yields").register(registry);
- startScanCalls =
- Counter.builder(METRICS_SCAN_START).description("calls to start a scan / multiscan")
- .tags(MetricsUtil.getCommonTags()).register(registry);
- continueScanCalls =
- Counter.builder(METRICS_SCAN_CONTINUE).description("calls to continue a scan / multiscan")
- .tags(MetricsUtil.getCommonTags()).register(registry);
- closeScanCalls =
- Counter.builder(METRICS_SCAN_CLOSE).description("calls to close a scan / multiscan")
- .tags(MetricsUtil.getCommonTags()).register(registry);
+ startScanCalls = Counter.builder(METRICS_SCAN_START)
+ .description("calls to start a scan / multiscan").register(registry);
+ continueScanCalls = Counter.builder(METRICS_SCAN_CONTINUE)
+ .description("calls to continue a scan / multiscan").register(registry);
+ closeScanCalls = Counter.builder(METRICS_SCAN_CLOSE)
+ .description("calls to close a scan / multiscan").register(registry);
busyTimeoutReturned = Counter.builder(METRICS_SCAN_BUSY_TIMEOUT)
- .description("times that a scan has timed out in the queue")
- .tags(MetricsUtil.getCommonTags()).register(registry);
+ .description("times that a scan has timed out in the queue").register(registry);
Gauge.builder(METRICS_TSERVER_QUERIES, this, TabletServerScanMetrics::getLookupCount)
.description("Number of queries").register(registry);
Gauge.builder(METRICS_TSERVER_SCAN_RESULTS, this, TabletServerScanMetrics::getQueryResultCount)
diff --git a/test/src/main/java/org/apache/accumulo/test/conf/store/PropCacheCaffeineImplZkIT.java b/test/src/main/java/org/apache/accumulo/test/conf/store/PropCacheCaffeineImplZkIT.java
index 5abb7057d35..22d06b83a18 100644
--- a/test/src/main/java/org/apache/accumulo/test/conf/store/PropCacheCaffeineImplZkIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/conf/store/PropCacheCaffeineImplZkIT.java
@@ -41,13 +41,11 @@
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
-import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.codec.VersionedPropCodec;
import org.apache.accumulo.server.conf.codec.VersionedProperties;
import org.apache.accumulo.server.conf.store.TablePropKey;
import org.apache.accumulo.server.conf.store.impl.PropCacheCaffeineImpl;
-import org.apache.accumulo.server.conf.store.impl.PropStoreMetrics;
import org.apache.accumulo.server.conf.store.impl.PropStoreWatcher;
import org.apache.accumulo.server.conf.store.impl.ReadyMonitor;
import org.apache.accumulo.server.conf.store.impl.ZooPropLoader;
@@ -79,7 +77,6 @@ public class PropCacheCaffeineImplZkIT {
private final TableId tIdA = TableId.of("A");
private final TableId tIdB = TableId.of("B");
- private final PropStoreMetrics cacheMetrics = new PropStoreMetrics();
private static ServerContext context;
@TempDir
@@ -161,12 +158,9 @@ public void init() throws Exception {
PropStoreWatcher propStoreWatcher = new PropStoreWatcher(readyMonitor);
- MetricsUtil.initializeProducers(cacheMetrics);
-
ZooPropLoader propLoader =
- new ZooPropLoader(zrw, VersionedPropCodec.getDefault(), propStoreWatcher, cacheMetrics);
- PropCacheCaffeineImpl cache =
- new PropCacheCaffeineImpl.Builder(propLoader, cacheMetrics).build();
+ new ZooPropLoader(zrw, VersionedPropCodec.getDefault(), propStoreWatcher);
+ PropCacheCaffeineImpl cache = new PropCacheCaffeineImpl.Builder(propLoader).build();
VersionedProperties readProps = cache.get(propStoreKey);
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index 61c3c24e7c7..1ad1141fa1d 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@ -124,7 +124,7 @@ public static void main(String[] args) throws Exception {
ServerAddress serverPort = TServerUtils.startTServer(context.getConfiguration(),
ThriftServerType.CUSTOM_HS_HA, muxProcessor, "ZombieTServer", "walking dead", 2,
ThreadPools.DEFAULT_TIMEOUT_MILLISECS, 1000, 10 * 1024 * 1024, null, null, -1,
- context.getConfiguration().getCount(Property.RPC_BACKLOG),
+ context.getConfiguration().getCount(Property.RPC_BACKLOG), context.getMetricsInfo(),
HostAndPort.fromParts("0.0.0.0", port));
String addressString = serverPort.address.toString();
diff --git a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
index df872a3a9ca..4c8c3bea26a 100644
--- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
@@ -43,6 +43,7 @@
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metrics.MetricsProducer;
+import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.test.functional.ConfigurableMacBase;
import org.apache.accumulo.test.metrics.TestStatsDSink.Metric;
@@ -79,11 +80,14 @@ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSit
cfg.setProperty(Property.GC_CYCLE_START, "1s");
cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
cfg.setProperty(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL, "1s");
-
- // Tell the server processes to use a StatsDMeterRegistry that will be configured
- // to push all metrics to the sink we started.
+ // Tell the server processes to use a StatsDMeterRegistry and the simple logging registry
+ // that will be configured to push all metrics to the sink we started.
cfg.setProperty(Property.GENERAL_MICROMETER_ENABLED, "true");
- cfg.setProperty(Property.GENERAL_MICROMETER_FACTORY, TestStatsDRegistryFactory.class.getName());
+ cfg.setProperty(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED, "true");
+ cfg.setProperty("general.custom.metrics.opts.logging.step", "1s");
+ String clazzList = LoggingMeterRegistryFactory.class.getName() + ","
+ + TestStatsDRegistryFactory.class.getName();
+ cfg.setProperty(Property.GENERAL_MICROMETER_FACTORY, clazzList);
Map sysProps = Map.of(TestStatsDRegistryFactory.SERVER_HOST, "127.0.0.1",
TestStatsDRegistryFactory.SERVER_PORT, Integer.toString(sink.getPort()));
cfg.setSystemProperties(sysProps);
@@ -97,9 +101,7 @@ public void confirmMetricsPublished() throws Exception {
Set unexpectedMetrics = Set.of(METRICS_SCAN_YIELDS, METRICS_UPDATE_ERRORS,
METRICS_REPLICATION_QUEUE, METRICS_COMPACTOR_MAJC_STUCK, METRICS_SCAN_BUSY_TIMEOUT);
- Set flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS, METRICS_FATE_TYPE_IN_PROGRESS,
- METRICS_PROPSTORE_EVICTION_COUNT, METRICS_PROPSTORE_REFRESH_COUNT,
- METRICS_PROPSTORE_REFRESH_LOAD_COUNT, METRICS_PROPSTORE_ZK_ERROR_COUNT);
+ Set flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS, METRICS_FATE_TYPE_IN_PROGRESS);
Map expectedMetricNames = this.getMetricFields();
flakyMetrics.forEach(expectedMetricNames::remove); // might not see these
diff --git a/test/src/main/java/org/apache/accumulo/test/metrics/TestStatsDRegistryFactory.java b/test/src/main/java/org/apache/accumulo/test/metrics/TestStatsDRegistryFactory.java
index 6c26fabb4d8..8715a40c00d 100644
--- a/test/src/main/java/org/apache/accumulo/test/metrics/TestStatsDRegistryFactory.java
+++ b/test/src/main/java/org/apache/accumulo/test/metrics/TestStatsDRegistryFactory.java
@@ -20,7 +20,7 @@
import java.time.Duration;
-import org.apache.accumulo.core.metrics.MeterRegistryFactory;
+import org.apache.accumulo.core.spi.metrics.MeterRegistryFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,8 +38,8 @@ public class TestStatsDRegistryFactory implements MeterRegistryFactory {
public static final String SERVER_PORT = "test.meter.registry.port";
@Override
- public MeterRegistry create() {
-
+ public MeterRegistry create(final InitParameters params) {
+ LOG.info("starting metrics registration.");
String host = System.getProperty(SERVER_HOST, null);
String port = System.getProperty(SERVER_PORT, null);
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
index 57fcddf1c6a..67a689c4f51 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
@@ -346,7 +346,7 @@ public static void main(String[] args) throws Exception {
TServerUtils.startTServer(context.getConfiguration(), ThriftServerType.CUSTOM_HS_HA,
muxProcessor, "NullTServer", "null tserver", 2, ThreadPools.DEFAULT_TIMEOUT_MILLISECS, 1000,
10 * 1024 * 1024, null, null, -1, context.getConfiguration().getCount(Property.RPC_BACKLOG),
- HostAndPort.fromParts("0.0.0.0", opts.port));
+ context.getMetricsInfo(), HostAndPort.fromParts("0.0.0.0", opts.port));
HostAndPort addr = HostAndPort.fromParts(InetAddress.getLocalHost().getHostName(), opts.port);
diff --git a/test/src/main/resources/log4j2-test.properties b/test/src/main/resources/log4j2-test.properties
index f3a23b74740..13ef01cdf37 100644
--- a/test/src/main/resources/log4j2-test.properties
+++ b/test/src/main/resources/log4j2-test.properties
@@ -152,29 +152,6 @@ logger.38.level = debug
logger.39.name = org.apache.accumulo.manager.Manager
logger.39.level = trace
-property.metricsFilename = ./target/test-metrics
-
-# appender.metrics.type = Console
-appender.metrics.type = RollingFile
-appender.metrics.name = LoggingMetricsOutput
-appender.metrics.fileName = ${metricsFilename}.metrics
-appender.metrics.filePattern = ${metricsFilename}-%d{MM-dd-yy-HH}-%i.metrics.gz
-appender.metrics.layout.type = PatternLayout
-appender.metrics.layout.pattern = METRICS: %d{ISO8601}, %m%n
-appender.metrics.policies.type = Policies
-appender.metrics.policies.time.type = TimeBasedTriggeringPolicy
-appender.metrics.policies.time.interval = 1
-appender.metrics.policies.time.modulate = true
-appender.metrics.policies.size.type = SizeBasedTriggeringPolicy
-appender.metrics.policies.size.size=100MB
-appender.metrics.strategy.type = DefaultRolloverStrategy
-appender.metrics.strategy.max = 10
-
-logger.metrics.name = org.apache.accumulo.metrics
-logger.metrics.level = info
-logger.metrics.additivity = false
-logger.metrics.appenderRef.metrics.ref = LoggingMetricsOutput
-
rootLogger.level = debug
rootLogger.appenderRef.console.ref = STDOUT