From 5eccb238849c0d54962a6f7f2d5a650424ce17c5 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 21 Jun 2022 14:08:25 +0530 Subject: [PATCH 01/47] feat: initial hit and trial for publishing reader metrics [https://github.com/odpf/dagger/issues/108] --- .../metrics/aspects/ParquetReaderAspects.java | 26 ++++++++++ .../source/parquet/reader/ParquetReader.java | 7 +++ .../parquet/reader/ParquetReaderMetrics.java | 51 +++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderMetrics.java diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java new file mode 100644 index 000000000..a79ce6675 --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java @@ -0,0 +1,26 @@ +package io.odpf.dagger.core.metrics.aspects; + +import io.odpf.dagger.common.metrics.aspects.AspectType; +import io.odpf.dagger.common.metrics.aspects.Aspects; + +public enum ParquetReaderAspects implements Aspects { + READER_CREATED("reader_created", AspectType.Metric); + + private String value; + private AspectType aspectType; + + ParquetReaderAspects(String value, AspectType aspectType) { + this.value = value; + this.aspectType = aspectType; + } + + @Override + public String getValue() { + return value; + } + + @Override + public AspectType getAspectType() { + return aspectType; + } +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java index 573c1b859..271c4324f 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java @@ -2,6 +2,7 @@ import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.ParquetFileSourceReaderInitializationException; +import io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects; import org.apache.flink.connector.file.src.reader.FileRecordFormat; import org.apache.flink.connector.file.src.util.CheckpointedPosition; import org.apache.flink.types.Row; @@ -33,6 +34,7 @@ public class ParquetReader implements FileRecordFormat.Reader { private RecordReader recordReader; private final MessageType schema; private long totalEmittedRowCount; + private final ParquetReaderMetrics parquetReaderMetrics; private static final Logger LOGGER = LoggerFactory.getLogger(ParquetReader.class.getName()); private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDeserializer, ParquetFileReader parquetFileReader) throws IOException { @@ -42,6 +44,9 @@ private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDe this.schema = this.parquetFileReader.getFileMetaData().getSchema(); this.isRecordReaderInitialized = false; this.totalEmittedRowCount = 0L; + this.parquetReaderMetrics = new ParquetReaderMetrics(); + this.parquetReaderMetrics.enqueueMetric(ParquetReaderAspects.READER_CREATED); + LOGGER.info("Constructor for ParquetReader called."); } private boolean checkIfNullPage(PageReadStore page) { @@ -82,6 +87,7 @@ private Row readRecords() throws IOException { @Nullable @Override public Row read() throws IOException { + parquetReaderMetrics.flushAll(); if (!isRecordReaderInitialized) { initializeRecordReader(); } @@ -98,6 +104,7 @@ private Row readAndDeserialize() { @Override public void close() throws IOException { + parquetReaderMetrics.flushAll(); parquetFileReader.close(); closeRecordReader(); String logMessage = String.format("Closed the ParquetFileReader and de-referenced the RecordReader for file %s", hadoopFilePath.getName()); diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderMetrics.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderMetrics.java new file mode 100644 index 000000000..3b7d32f20 --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderMetrics.java @@ -0,0 +1,51 @@ +package io.odpf.dagger.core.source.parquet.reader; + +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects; +import org.apache.flink.api.common.functions.AbstractRichFunction; +import org.apache.flink.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.LinkedList; + +public class ParquetReaderMetrics extends AbstractRichFunction { + private MeterStatsManager meterStatsManager; + private final LinkedList metricsToPublishQueue; + private static final Logger LOGGER = LoggerFactory.getLogger(ParquetReaderMetrics.class.getName()); + + public ParquetReaderMetrics() { + metricsToPublishQueue = new LinkedList<>(); + LOGGER.info("Initialized ParquetReaderMetrics class."); + } + + @Override + public void open(Configuration parameters) throws Exception { + LOGGER.info("Lifecycle function open called in ParquetReaderMetrics class."); + if (meterStatsManager == null) { + LOGGER.info("MeterStatsManager is null. Creating one."); + meterStatsManager = new MeterStatsManager(getRuntimeContext().getMetricGroup(), true); + LOGGER.info("Calling flushAll from within open in ParquetReaderMetrics class."); + flushAll(); + } + } + + public void enqueueMetric(ParquetReaderAspects aspect) { + LOGGER.info("Adding metric to queue:" + aspect.getValue()); + metricsToPublishQueue.addLast(aspect); + if (meterStatsManager != null) { + LOGGER.info("MeterStatsManager is not null. Flushing metric after enqueuing:" + aspect.getValue()); + flushAll(); + } + } + + public void flushAll() { + LOGGER.info("Preparing to flush metrics from queue."); + while (!metricsToPublishQueue.isEmpty()) { + ParquetReaderAspects aspect = metricsToPublishQueue.poll(); + System.out.println("Publishing metric " + aspect.getValue()); + meterStatsManager.markEvent(aspect); + } + LOGGER.info("All metrics in queue flushed."); + } +} From 7f267457755a9cbaf3eb766b25bbb917531e7b85 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 21 Jun 2022 14:10:45 +0530 Subject: [PATCH 02/47] Revert "feat: initial hit and trial for publishing reader metrics" This reverts commit 5eccb238849c0d54962a6f7f2d5a650424ce17c5. --- .../metrics/aspects/ParquetReaderAspects.java | 26 ---------- .../source/parquet/reader/ParquetReader.java | 7 --- .../parquet/reader/ParquetReaderMetrics.java | 51 ------------------- 3 files changed, 84 deletions(-) delete mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java delete mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderMetrics.java diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java deleted file mode 100644 index a79ce6675..000000000 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.odpf.dagger.core.metrics.aspects; - -import io.odpf.dagger.common.metrics.aspects.AspectType; -import io.odpf.dagger.common.metrics.aspects.Aspects; - -public enum ParquetReaderAspects implements Aspects { - READER_CREATED("reader_created", AspectType.Metric); - - private String value; - private AspectType aspectType; - - ParquetReaderAspects(String value, AspectType aspectType) { - this.value = value; - this.aspectType = aspectType; - } - - @Override - public String getValue() { - return value; - } - - @Override - public AspectType getAspectType() { - return aspectType; - } -} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java index 271c4324f..573c1b859 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java @@ -2,7 +2,6 @@ import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.ParquetFileSourceReaderInitializationException; -import io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects; import org.apache.flink.connector.file.src.reader.FileRecordFormat; import org.apache.flink.connector.file.src.util.CheckpointedPosition; import org.apache.flink.types.Row; @@ -34,7 +33,6 @@ public class ParquetReader implements FileRecordFormat.Reader { private RecordReader recordReader; private final MessageType schema; private long totalEmittedRowCount; - private final ParquetReaderMetrics parquetReaderMetrics; private static final Logger LOGGER = LoggerFactory.getLogger(ParquetReader.class.getName()); private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDeserializer, ParquetFileReader parquetFileReader) throws IOException { @@ -44,9 +42,6 @@ private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDe this.schema = this.parquetFileReader.getFileMetaData().getSchema(); this.isRecordReaderInitialized = false; this.totalEmittedRowCount = 0L; - this.parquetReaderMetrics = new ParquetReaderMetrics(); - this.parquetReaderMetrics.enqueueMetric(ParquetReaderAspects.READER_CREATED); - LOGGER.info("Constructor for ParquetReader called."); } private boolean checkIfNullPage(PageReadStore page) { @@ -87,7 +82,6 @@ private Row readRecords() throws IOException { @Nullable @Override public Row read() throws IOException { - parquetReaderMetrics.flushAll(); if (!isRecordReaderInitialized) { initializeRecordReader(); } @@ -104,7 +98,6 @@ private Row readAndDeserialize() { @Override public void close() throws IOException { - parquetReaderMetrics.flushAll(); parquetFileReader.close(); closeRecordReader(); String logMessage = String.format("Closed the ParquetFileReader and de-referenced the RecordReader for file %s", hadoopFilePath.getName()); diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderMetrics.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderMetrics.java deleted file mode 100644 index 3b7d32f20..000000000 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderMetrics.java +++ /dev/null @@ -1,51 +0,0 @@ -package io.odpf.dagger.core.source.parquet.reader; - -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; -import io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects; -import org.apache.flink.api.common.functions.AbstractRichFunction; -import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.LinkedList; - -public class ParquetReaderMetrics extends AbstractRichFunction { - private MeterStatsManager meterStatsManager; - private final LinkedList metricsToPublishQueue; - private static final Logger LOGGER = LoggerFactory.getLogger(ParquetReaderMetrics.class.getName()); - - public ParquetReaderMetrics() { - metricsToPublishQueue = new LinkedList<>(); - LOGGER.info("Initialized ParquetReaderMetrics class."); - } - - @Override - public void open(Configuration parameters) throws Exception { - LOGGER.info("Lifecycle function open called in ParquetReaderMetrics class."); - if (meterStatsManager == null) { - LOGGER.info("MeterStatsManager is null. Creating one."); - meterStatsManager = new MeterStatsManager(getRuntimeContext().getMetricGroup(), true); - LOGGER.info("Calling flushAll from within open in ParquetReaderMetrics class."); - flushAll(); - } - } - - public void enqueueMetric(ParquetReaderAspects aspect) { - LOGGER.info("Adding metric to queue:" + aspect.getValue()); - metricsToPublishQueue.addLast(aspect); - if (meterStatsManager != null) { - LOGGER.info("MeterStatsManager is not null. Flushing metric after enqueuing:" + aspect.getValue()); - flushAll(); - } - } - - public void flushAll() { - LOGGER.info("Preparing to flush metrics from queue."); - while (!metricsToPublishQueue.isEmpty()) { - ParquetReaderAspects aspect = metricsToPublishQueue.poll(); - System.out.println("Publishing metric " + aspect.getValue()); - meterStatsManager.markEvent(aspect); - } - LOGGER.info("All metrics in queue flushed."); - } -} From 9d0c8e847d93fca66ec289e5df25e32d322711be Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 21 Jun 2022 14:32:12 +0530 Subject: [PATCH 03/47] feat: check if open is called [https://github.com/odpf/dagger/issues/108] --- .../dagger/core/source/parquet/reader/ParquetReader.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java index 573c1b859..0903762c1 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java @@ -2,6 +2,7 @@ import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.ParquetFileSourceReaderInitializationException; +import org.apache.flink.api.common.functions.AbstractRichFunction; import org.apache.flink.connector.file.src.reader.FileRecordFormat; import org.apache.flink.connector.file.src.util.CheckpointedPosition; import org.apache.flink.types.Row; @@ -23,7 +24,7 @@ import javax.annotation.Nullable; import java.io.IOException; -public class ParquetReader implements FileRecordFormat.Reader { +public class ParquetReader extends AbstractRichFunction implements FileRecordFormat.Reader { private final Path hadoopFilePath; private final SimpleGroupDeserializer simpleGroupDeserializer; private long currentRecordIndex; @@ -42,6 +43,12 @@ private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDe this.schema = this.parquetFileReader.getFileMetaData().getSchema(); this.isRecordReaderInitialized = false; this.totalEmittedRowCount = 0L; + LOGGER.info("Constructor called in ParquetReader class"); + } + + @Override + public void open(org.apache.flink.configuration.Configuration internalFlinkConfig) throws Exception { + LOGGER.info("Lifecycle function open called in ParquetReader class"); } private boolean checkIfNullPage(PageReadStore page) { From d196fb0c75caca1d4aa7cda1cee8f40619196dc7 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 21 Jun 2022 16:55:18 +0530 Subject: [PATCH 04/47] Revert "feat: check if open is called" This reverts commit 9d0c8e847d93fca66ec289e5df25e32d322711be. --- .../dagger/core/source/parquet/reader/ParquetReader.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java index 0903762c1..573c1b859 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java @@ -2,7 +2,6 @@ import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.ParquetFileSourceReaderInitializationException; -import org.apache.flink.api.common.functions.AbstractRichFunction; import org.apache.flink.connector.file.src.reader.FileRecordFormat; import org.apache.flink.connector.file.src.util.CheckpointedPosition; import org.apache.flink.types.Row; @@ -24,7 +23,7 @@ import javax.annotation.Nullable; import java.io.IOException; -public class ParquetReader extends AbstractRichFunction implements FileRecordFormat.Reader { +public class ParquetReader implements FileRecordFormat.Reader { private final Path hadoopFilePath; private final SimpleGroupDeserializer simpleGroupDeserializer; private long currentRecordIndex; @@ -43,12 +42,6 @@ private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDe this.schema = this.parquetFileReader.getFileMetaData().getSchema(); this.isRecordReaderInitialized = false; this.totalEmittedRowCount = 0L; - LOGGER.info("Constructor called in ParquetReader class"); - } - - @Override - public void open(org.apache.flink.configuration.Configuration internalFlinkConfig) throws Exception { - LOGGER.info("Lifecycle function open called in ParquetReader class"); } private boolean checkIfNullPage(PageReadStore page) { From acceed977dd1e66b2d0bcbbe7ff7eb8a17341339 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 24 Jun 2022 18:15:08 +0530 Subject: [PATCH 05/47] feat: refactor package names - add new package types for flink and statsd measurements [https://github.com/odpf/dagger/issues/108] --- .../dagger/common/metrics/type/Counter.java | 11 +++++++ .../dagger/common/metrics/type/Gauge.java | 13 +++++++++ .../metrics/type/MeasurementManager.java | 10 +++++++ .../dagger/common/metrics/type/Meter.java | 11 +++++++ .../flink}/CounterStatsManager.java | 2 +- .../flink}/GaugeStatsManager.java | 2 +- .../flink}/MeterStatsManager.java | 2 +- .../type/statsd/DaggerCounterManager.java | 24 +++++++++++++++ .../type/statsd/DaggerGaugeManager.java | 29 +++++++++++++++++++ .../type/statsd/DaggerMeterManager.java | 24 +++++++++++++++ .../odpf/dagger/common/udfs/AggregateUdf.java | 2 +- .../io/odpf/dagger/common/udfs/ScalarUdf.java | 2 +- .../io/odpf/dagger/common/udfs/TableUdf.java | 2 +- .../flink}/CounterStatsManagerTest.java | 4 +-- .../flink}/GaugeStatsManagerTest.java | 4 +-- .../flink}/MeterStatsManagerTest.java | 4 +-- .../{managers => }/utils/TestAspects.java | 2 +- .../processors/common/EndpointHandler.java | 2 +- .../common/PostResponseTelemetry.java | 2 +- .../processors/external/AsyncConnector.java | 2 +- .../external/es/EsAsyncConnector.java | 2 +- .../external/es/EsResponseHandler.java | 2 +- .../external/grpc/GrpcAsyncConnector.java | 2 +- .../external/grpc/GrpcResponseHandler.java | 2 +- .../external/http/HttpAsyncConnector.java | 2 +- .../external/http/HttpResponseHandler.java | 2 +- .../external/pg/PgAsyncConnector.java | 2 +- .../external/pg/PgResponseHandler.java | 2 +- .../longbow/processor/LongbowReader.java | 2 +- .../longbow/processor/LongbowWriter.java | 2 +- .../processor/MetricsTelemetryExporter.java | 2 +- .../common/EndpointHandlerTest.java | 2 +- .../external/es/EsAsyncConnectorTest.java | 2 +- .../external/es/EsResponseHandlerTest.java | 2 +- .../external/grpc/GrpcAsyncConnectorTest.java | 2 +- .../grpc/GrpcResponseHandlerTest.java | 2 +- .../external/http/HttpAsyncConnectorTest.java | 2 +- .../http/HttpResponseHandlerTest.java | 2 +- .../external/pg/PgAsyncConnectorTest.java | 2 +- .../external/pg/PgResponseHandlerTest.java | 2 +- .../longbow/processor/LongbowReaderTest.java | 2 +- .../longbow/processor/LongbowWriterTest.java | 2 +- .../MetricsTelemetryExporterTest.java | 2 +- .../InvalidRecordFilterTransformer.java | 2 +- .../functions/udfs/scalar/DartContains.java | 2 +- .../dagger/functions/udfs/scalar/DartGet.java | 2 +- .../udfs/scalar/dart/store/gcs/GcsClient.java | 2 +- .../scalar/dart/store/gcs/GcsDataStore.java | 4 +-- .../udfs/scalar/DartContainsTest.java | 2 +- .../functions/udfs/scalar/DartGetTest.java | 2 +- .../dart/store/gcs/GcsDataStoreTest.java | 2 +- 51 files changed, 170 insertions(+), 48 deletions(-) create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Counter.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java rename dagger-common/src/main/java/io/odpf/dagger/common/metrics/{managers => type/flink}/CounterStatsManager.java (97%) rename dagger-common/src/main/java/io/odpf/dagger/common/metrics/{managers => type/flink}/GaugeStatsManager.java (98%) rename dagger-common/src/main/java/io/odpf/dagger/common/metrics/{managers => type/flink}/MeterStatsManager.java (98%) create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerCounterManager.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerGaugeManager.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerMeterManager.java rename dagger-common/src/test/java/io/odpf/dagger/common/metrics/{managers => type/flink}/CounterStatsManagerTest.java (96%) rename dagger-common/src/test/java/io/odpf/dagger/common/metrics/{managers => type/flink}/GaugeStatsManagerTest.java (94%) rename dagger-common/src/test/java/io/odpf/dagger/common/metrics/{managers => type/flink}/MeterStatsManagerTest.java (96%) rename dagger-common/src/test/java/io/odpf/dagger/common/metrics/{managers => }/utils/TestAspects.java (92%) diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Counter.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Counter.java new file mode 100644 index 000000000..db4b43a2a --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Counter.java @@ -0,0 +1,11 @@ +package io.odpf.dagger.common.metrics.type; + +import io.odpf.dagger.common.metrics.aspects.Aspects; + +import java.io.Serializable; + +public interface Counter extends Serializable { + void increment(Aspects aspect); + + void decrement(Aspects aspect); +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java new file mode 100644 index 000000000..ef777e782 --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java @@ -0,0 +1,13 @@ +package io.odpf.dagger.common.metrics.type; + +import io.odpf.dagger.common.metrics.aspects.Aspects; + +import java.io.Serializable; + +public interface Gauge extends Serializable { + void registerInteger(Aspects aspect, int gaugeValue); + + void registerString(Aspects aspect, String gaugeValue); + + void registerDouble(Aspects aspect, double gaugeValue); +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java new file mode 100644 index 000000000..d622f8a8c --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java @@ -0,0 +1,10 @@ +package io.odpf.dagger.common.metrics.type; + +import io.odpf.dagger.common.metrics.aspects.Aspects; + +import java.io.Serializable; +import java.util.Map; + +public interface MeasurementManager extends Serializable { + void register(Aspects[] aspect, Map tagKeyValuePairs); +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java new file mode 100644 index 000000000..94fb4d70a --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java @@ -0,0 +1,11 @@ +package io.odpf.dagger.common.metrics.type; + +import io.odpf.dagger.common.metrics.aspects.Aspects; + +import java.io.Serializable; + +public interface Meter extends Serializable { + void markEvent(Aspects aspect); + + void markEvent(Aspects aspect, long num); +} \ No newline at end of file diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/CounterStatsManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManager.java similarity index 97% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/CounterStatsManager.java rename to dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManager.java index 8b924a1ae..2f4a86bba 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/CounterStatsManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManager.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.managers; +package io.odpf.dagger.common.metrics.type.flink; import io.odpf.dagger.common.metrics.aspects.AspectType; import io.odpf.dagger.common.metrics.aspects.Aspects; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManager.java similarity index 98% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManager.java rename to dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManager.java index 79d8d32e4..4e3eaf31e 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManager.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.managers; +package io.odpf.dagger.common.metrics.type.flink; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.MetricGroup; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/MeterStatsManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManager.java similarity index 98% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/MeterStatsManager.java rename to dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManager.java index 46de410d7..ffa5a8d37 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/MeterStatsManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManager.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.managers; +package io.odpf.dagger.common.metrics.type.flink; import org.apache.flink.dropwizard.metrics.DropwizardHistogramWrapper; import org.apache.flink.dropwizard.metrics.DropwizardMeterWrapper; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerCounterManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerCounterManager.java new file mode 100644 index 000000000..c4630c882 --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerCounterManager.java @@ -0,0 +1,24 @@ +package io.odpf.dagger.common.metrics.type.statsd; + +import io.odpf.dagger.common.metrics.aspects.Aspects; +import io.odpf.dagger.common.metrics.type.Counter; +import io.odpf.dagger.common.metrics.type.MeasurementManager; + +import java.util.Map; + +public class DaggerCounterManager implements MeasurementManager, Counter { + @Override + public void increment(Aspects aspect) { + + } + + @Override + public void decrement(Aspects aspect) { + + } + + @Override + public void register(Aspects[] aspect, Map tagKeyValuePairs) { + + } +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerGaugeManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerGaugeManager.java new file mode 100644 index 000000000..6299e760b --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerGaugeManager.java @@ -0,0 +1,29 @@ +package io.odpf.dagger.common.metrics.type.statsd; + +import io.odpf.dagger.common.metrics.aspects.Aspects; +import io.odpf.dagger.common.metrics.type.Gauge; +import io.odpf.dagger.common.metrics.type.MeasurementManager; + +import java.util.Map; + +public class DaggerGaugeManager implements MeasurementManager, Gauge { + @Override + public void registerInteger(Aspects aspect, int gaugeValue) { + + } + + @Override + public void registerString(Aspects aspect, String gaugeValue) { + + } + + @Override + public void registerDouble(Aspects aspect, double gaugeValue) { + + } + + @Override + public void register(Aspects[] aspect, Map tagKeyValuePairs) { + + } +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerMeterManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerMeterManager.java new file mode 100644 index 000000000..ba0520cf6 --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerMeterManager.java @@ -0,0 +1,24 @@ +package io.odpf.dagger.common.metrics.type.statsd; + +import io.odpf.dagger.common.metrics.aspects.Aspects; +import io.odpf.dagger.common.metrics.type.MeasurementManager; +import io.odpf.dagger.common.metrics.type.Meter; + +import java.util.Map; + +public class DaggerMeterManager implements MeasurementManager, Meter { + @Override + public void register(Aspects[] aspect, Map tagKeyValuePairs) { + + } + + @Override + public void markEvent(Aspects aspect) { + + } + + @Override + public void markEvent(Aspects aspect, long num) { + + } +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java index e7242c8a9..c1defe40a 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.udfs; -import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; +import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; import org.apache.flink.table.functions.AggregateFunction; import org.apache.flink.table.functions.FunctionContext; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java index 83685f25e..4c04fc58e 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.udfs; -import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; +import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.functions.ScalarFunction; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java index ae5b3f28e..f0d0e3267 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.udfs; -import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; +import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.functions.TableFunction; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/CounterStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java similarity index 96% rename from dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/CounterStatsManagerTest.java rename to dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java index 4a96300d9..0d45cf615 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/CounterStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java @@ -1,6 +1,6 @@ -package io.odpf.dagger.common.metrics.managers; +package io.odpf.dagger.common.metrics.type.flink; -import io.odpf.dagger.common.metrics.managers.utils.TestAspects; +import io.odpf.dagger.common.metrics.utils.TestAspects; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.metrics.SimpleCounter; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java similarity index 94% rename from dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManagerTest.java rename to dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java index e88013116..16231a7cd 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java @@ -1,6 +1,6 @@ -package io.odpf.dagger.common.metrics.managers; +package io.odpf.dagger.common.metrics.type.flink; -import io.odpf.dagger.common.metrics.managers.utils.TestAspects; +import io.odpf.dagger.common.metrics.utils.TestAspects; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.MetricGroup; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java similarity index 96% rename from dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java rename to dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java index 5a5c4134e..226809114 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java @@ -1,11 +1,11 @@ -package io.odpf.dagger.common.metrics.managers; +package io.odpf.dagger.common.metrics.type.flink; +import io.odpf.dagger.common.metrics.utils.TestAspects; import org.apache.flink.dropwizard.metrics.DropwizardMeterWrapper; import org.apache.flink.metrics.Histogram; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MetricGroup; -import io.odpf.dagger.common.metrics.managers.utils.TestAspects; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/utils/TestAspects.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/utils/TestAspects.java similarity index 92% rename from dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/utils/TestAspects.java rename to dagger-common/src/test/java/io/odpf/dagger/common/metrics/utils/TestAspects.java index bc2e02d2e..7234bf8b2 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/utils/TestAspects.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/utils/TestAspects.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.managers.utils; +package io.odpf.dagger.common.metrics.utils; import io.odpf.dagger.common.metrics.aspects.AspectType; import io.odpf.dagger.common.metrics.aspects.Aspects; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java index 22694b3c8..b24da4d37 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java @@ -7,7 +7,7 @@ import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; import io.odpf.dagger.core.exception.InvalidConfigurationException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.common.serde.typehandler.TypeHandler; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java index e8e479327..833367d13 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java @@ -1,6 +1,6 @@ package io.odpf.dagger.core.processors.common; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import java.time.Instant; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java index eb5622e79..43325e921 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java @@ -8,7 +8,7 @@ import com.google.protobuf.Descriptors; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java index 93e76a4bb..db6c36b65 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java @@ -1,6 +1,6 @@ package io.odpf.dagger.core.processors.external.es; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.common.PostResponseTelemetry; import io.odpf.dagger.core.processors.common.RowManager; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java index 09c0e613a..d02fde809 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java @@ -9,7 +9,7 @@ import com.jayway.jsonpath.PathNotFoundException; import io.odpf.dagger.common.serde.typehandler.TypeHandler; import io.odpf.dagger.common.serde.typehandler.TypeHandlerFactory; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.exception.HttpFailureException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java index 173be65f2..7e79d2c71 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java @@ -13,7 +13,7 @@ import io.odpf.dagger.core.exception.ChannelNotAvailableException; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; import io.odpf.dagger.core.exception.InvalidGrpcBodyException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.external.grpc.client.GrpcClient; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java index 12441ecb1..f204fc8ca 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.grpc; import io.odpf.dagger.core.exception.GrpcFailureException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.ColumnNameManager; import io.odpf.dagger.core.processors.common.OutputMapping; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java index fcb401904..8b9fd57aa 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.http; import io.odpf.dagger.core.exception.InvalidHttpVerbException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.common.DescriptorManager; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java index 5b84c4803..388406504 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.http; import io.odpf.dagger.core.exception.HttpFailureException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.ColumnNameManager; import io.odpf.dagger.core.processors.common.OutputMapping; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java index 46595164e..e81166916 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.pg; import io.odpf.dagger.core.exception.InvalidConfigurationException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.common.PostResponseTelemetry; import io.odpf.dagger.core.processors.common.RowManager; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java index bcfd9cec8..4e7b90984 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.pg; import io.odpf.dagger.core.exception.HttpFailureException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.ColumnNameManager; import io.odpf.dagger.core.processors.common.PostResponseTelemetry; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java index 381abf73a..452bc6a00 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java @@ -5,7 +5,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.LongbowReaderAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.reporters.ErrorReporterFactory; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java index bd02ac9c9..1a01c64a0 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java @@ -5,7 +5,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.LongbowWriterAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.reporters.ErrorReporterFactory; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java index 73f5c97c7..f26e18289 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java @@ -6,7 +6,7 @@ import io.odpf.dagger.core.metrics.telemetry.TelemetryPublisher; import io.odpf.dagger.core.metrics.telemetry.TelemetrySubscriber; -import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; +import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; import io.odpf.dagger.core.metrics.aspects.TelemetryAspects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java index e103815c6..c40d28e56 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java @@ -4,7 +4,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.consumer.TestEnumType; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java index ade3dd94c..21c1dd594 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java @@ -7,7 +7,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java index 80b52ebcd..8340c6d0a 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java @@ -2,7 +2,7 @@ import com.google.protobuf.Descriptors; import com.jayway.jsonpath.PathNotFoundException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestEnrichedBookingLogMessage; import io.odpf.dagger.consumer.TestProfile; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java index 7b9c04857..8cb9d0124 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java @@ -4,7 +4,7 @@ import com.google.protobuf.DynamicMessage; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestGrpcRequest; import io.odpf.dagger.core.exception.InvalidConfigurationException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java index 4218c95f6..4d7d7974c 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java @@ -5,7 +5,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.jayway.jsonpath.PathNotFoundException; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestGrpcResponse; import io.odpf.dagger.consumer.TestLocation; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java index 2f401b59d..d2f8c32ab 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java @@ -3,7 +3,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.exception.InvalidHttpVerbException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java index c5fc30f84..4e1cec53b 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java @@ -2,7 +2,7 @@ import com.google.protobuf.Descriptors; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestSurgeFactorLogMessage; import io.odpf.dagger.core.exception.HttpFailureException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java index ec71872d5..2aa5a2e78 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java @@ -3,7 +3,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java index 693060719..6de7ab52e 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java @@ -2,7 +2,7 @@ import com.google.protobuf.Descriptors; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestSurgeFactorLogMessage; import io.odpf.dagger.core.exception.HttpFailureException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java index ea59255cc..345209ef3 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java @@ -4,7 +4,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.telemetry.TelemetrySubscriber; import io.odpf.dagger.core.processors.longbow.LongbowSchema; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java index da9fe73aa..5e9561cbc 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java @@ -5,7 +5,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.LongbowWriterAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.telemetry.TelemetrySubscriber; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java index 72cf3242e..c71b827ad 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java @@ -6,7 +6,7 @@ import org.apache.flink.metrics.groups.OperatorMetricGroup; import org.apache.flink.types.Row; -import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; +import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; import io.odpf.dagger.core.metrics.telemetry.TelemetryPublisher; import org.junit.Before; import org.junit.Test; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java index 622b4e0c8..52a652e20 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java @@ -7,7 +7,7 @@ import io.odpf.dagger.common.configuration.Configuration; import io.odpf.dagger.common.core.StreamInfo; import io.odpf.dagger.common.core.Transformer; -import io.odpf.dagger.common.metrics.managers.CounterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.CounterStatsManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java index ba817562f..0ad94d908 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java @@ -1,6 +1,6 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.common.udfs.ScalarUdf; import io.odpf.dagger.functions.udfs.scalar.dart.DartAspects; import io.odpf.dagger.functions.udfs.scalar.dart.store.gcs.GcsDataStore; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java index b35ddf4c0..914b26ebf 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java @@ -1,6 +1,6 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.common.udfs.ScalarUdf; import io.odpf.dagger.functions.exceptions.KeyDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.DartAspects; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java index 208580f05..d9d935cf8 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java @@ -6,7 +6,7 @@ import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; -import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; +import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; import io.odpf.dagger.functions.exceptions.BucketDoesNotExistException; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java index 20648c2ba..9696b2c2b 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java @@ -1,7 +1,7 @@ package io.odpf.dagger.functions.udfs.scalar.dart.store.gcs; -import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.functions.udfs.scalar.DartContains; import io.odpf.dagger.functions.udfs.scalar.DartGet; import io.odpf.dagger.functions.udfs.scalar.dart.DartAspects; diff --git a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java index 3ea29af89..8ccb880c7 100644 --- a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java +++ b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java @@ -1,7 +1,7 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.store.gcs.GcsDataStore; import io.odpf.dagger.functions.udfs.scalar.dart.types.SetCache; diff --git a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java index ee6072ba5..74747f4fd 100644 --- a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java +++ b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java @@ -1,6 +1,6 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.functions.exceptions.KeyDoesNotExistException; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.store.gcs.GcsDataStore; diff --git a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java index 95a4c5f5e..a864e3239 100644 --- a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java +++ b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java @@ -1,7 +1,7 @@ package io.odpf.dagger.functions.udfs.scalar.dart.store.gcs; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; +import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; import io.odpf.dagger.functions.exceptions.BucketDoesNotExistException; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.types.MapCache; From 579139e7bd863e346bcb47f3feb1e378d1a6f868 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Mon, 27 Jun 2022 19:45:44 +0530 Subject: [PATCH 06/47] feat: exclude dogstatsd from stencil and use custom version - Dagger will use its own version to send metrics to telegraf. Keeping multiple versions of the same dependency can be avoided here. [https://github.com/odpf/dagger/issues/108] --- dagger-common/build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dagger-common/build.gradle b/dagger-common/build.gradle index a41ff942f..62188c414 100644 --- a/dagger-common/build.gradle +++ b/dagger-common/build.gradle @@ -66,7 +66,10 @@ dependencies { dependenciesCommonJar 'org.apache.flink:flink-metrics-dropwizard:' + flinkVersion dependenciesCommonJar 'org.apache.flink:flink-json:' + flinkVersion dependenciesCommonJar 'com.jayway.jsonpath:json-path:2.4.0' - dependenciesCommonJar 'io.odpf:stencil:0.1.6' + dependenciesCommonJar('io.odpf:stencil:0.1.6') { + exclude group: 'com.datadoghq', module: 'java-dogstatsd-client' + } + dependenciesCommonJar 'com.datadoghq:java-dogstatsd-client:4.0.0' dependenciesCommonJar 'com.google.code.gson:gson:2.8.2' dependenciesCommonJar 'org.apache.parquet:parquet-column:1.12.2' From ffb60ec3557007515be1152d3d3dd2a0b743fec9 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Mon, 27 Jun 2022 19:46:51 +0530 Subject: [PATCH 07/47] feat: implement metrics via dogstatsd [https://github.com/odpf/dagger/issues/108] --- .../dagger/common/metrics/type/Gauge.java | 2 +- .../dagger/common/metrics/type/Histogram.java | 11 ++++ .../metrics/type/MeasurementManager.java | 5 +- .../dagger/common/metrics/type/Meter.java | 2 +- .../type/statsd/DaggerCounterManager.java | 24 -------- .../type/statsd/DaggerGaugeManager.java | 29 ---------- .../type/statsd/DaggerMeterManager.java | 24 -------- .../type/statsd/DaggerStatsDClient.java | 41 ++++++++++++++ .../SerializedStatsDClientSupplier.java | 10 ++++ .../statsd/manager/DaggerCounterManager.java | 43 +++++++++++++++ .../statsd/manager/DaggerGaugeManager.java | 48 ++++++++++++++++ .../manager/DaggerHistogramManager.java | 44 +++++++++++++++ .../statsd/manager/DaggerMeterManager.java | 44 +++++++++++++++ .../metrics/type/statsd/tags/StatsDTag.java | 34 ++++++++++++ .../io/odpf/dagger/core/StreamManager.java | 6 +- .../metrics/aspects/ParquetReaderAspects.java | 32 +++++++++++ .../telemetry/statsd/ComponentTags.java | 12 ++++ .../statsd/DaggerStatsDClientProvider.java | 41 ++++++++++++++ .../metrics/telemetry/statsd/GlobalTags.java | 5 ++ .../core/source/DaggerSourceFactory.java | 9 +-- .../io/odpf/dagger/core/source/Stream.java | 7 ++- .../dagger/core/source/StreamsFactory.java | 5 +- .../source/parquet/ParquetDaggerSource.java | 8 ++- .../parquet/ParquetFileRecordFormat.java | 15 ++++- .../source/parquet/reader/ParquetReader.java | 55 +++++++++++++++---- .../core/source/DaggerSourceFactoryTest.java | 13 +++-- .../odpf/dagger/core/source/StreamTest.java | 15 +++-- .../core/source/StreamsFactoryTest.java | 11 +++- .../parquet/ParquetDaggerSourceTest.java | 20 ++++--- .../parquet/ParquetFileRecordFormatTest.java | 9 +++ .../parquet/reader/ParquetReaderTest.java | 23 +++++--- 31 files changed, 512 insertions(+), 135 deletions(-) create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Histogram.java delete mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerCounterManager.java delete mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerGaugeManager.java delete mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerMeterManager.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerStatsDClient.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/SerializedStatsDClientSupplier.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerCounterManager.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerGaugeManager.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerMeterManager.java create mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/tags/StatsDTag.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/ComponentTags.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/GlobalTags.java diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java index ef777e782..72b7cd9ef 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java @@ -5,7 +5,7 @@ import java.io.Serializable; public interface Gauge extends Serializable { - void registerInteger(Aspects aspect, int gaugeValue); + void registerLong(Aspects aspect, long gaugeValue); void registerString(Aspects aspect, String gaugeValue); diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Histogram.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Histogram.java new file mode 100644 index 000000000..5a4e3894e --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Histogram.java @@ -0,0 +1,11 @@ +package io.odpf.dagger.common.metrics.type; + +import io.odpf.dagger.common.metrics.aspects.Aspects; + +import java.io.Serializable; + +public interface Histogram extends Serializable { + void recordValue(Aspects aspect, long value); + + void recordValue(Aspects aspect, double value); +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java index d622f8a8c..7c7044563 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java @@ -1,10 +1,11 @@ package io.odpf.dagger.common.metrics.type; import io.odpf.dagger.common.metrics.aspects.Aspects; +import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; import java.io.Serializable; -import java.util.Map; public interface MeasurementManager extends Serializable { - void register(Aspects[] aspect, Map tagKeyValuePairs); + void register(Aspects[] aspect, StatsDTag[] tags); + void register(StatsDTag[] tags); } diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java index 94fb4d70a..cdc7e62ed 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java @@ -8,4 +8,4 @@ public interface Meter extends Serializable { void markEvent(Aspects aspect); void markEvent(Aspects aspect, long num); -} \ No newline at end of file +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerCounterManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerCounterManager.java deleted file mode 100644 index c4630c882..000000000 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerCounterManager.java +++ /dev/null @@ -1,24 +0,0 @@ -package io.odpf.dagger.common.metrics.type.statsd; - -import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.Counter; -import io.odpf.dagger.common.metrics.type.MeasurementManager; - -import java.util.Map; - -public class DaggerCounterManager implements MeasurementManager, Counter { - @Override - public void increment(Aspects aspect) { - - } - - @Override - public void decrement(Aspects aspect) { - - } - - @Override - public void register(Aspects[] aspect, Map tagKeyValuePairs) { - - } -} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerGaugeManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerGaugeManager.java deleted file mode 100644 index 6299e760b..000000000 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerGaugeManager.java +++ /dev/null @@ -1,29 +0,0 @@ -package io.odpf.dagger.common.metrics.type.statsd; - -import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.Gauge; -import io.odpf.dagger.common.metrics.type.MeasurementManager; - -import java.util.Map; - -public class DaggerGaugeManager implements MeasurementManager, Gauge { - @Override - public void registerInteger(Aspects aspect, int gaugeValue) { - - } - - @Override - public void registerString(Aspects aspect, String gaugeValue) { - - } - - @Override - public void registerDouble(Aspects aspect, double gaugeValue) { - - } - - @Override - public void register(Aspects[] aspect, Map tagKeyValuePairs) { - - } -} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerMeterManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerMeterManager.java deleted file mode 100644 index ba0520cf6..000000000 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerMeterManager.java +++ /dev/null @@ -1,24 +0,0 @@ -package io.odpf.dagger.common.metrics.type.statsd; - -import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.MeasurementManager; -import io.odpf.dagger.common.metrics.type.Meter; - -import java.util.Map; - -public class DaggerMeterManager implements MeasurementManager, Meter { - @Override - public void register(Aspects[] aspect, Map tagKeyValuePairs) { - - } - - @Override - public void markEvent(Aspects aspect) { - - } - - @Override - public void markEvent(Aspects aspect, long num) { - - } -} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerStatsDClient.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerStatsDClient.java new file mode 100644 index 000000000..758ec40bf --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerStatsDClient.java @@ -0,0 +1,41 @@ +package io.odpf.dagger.common.metrics.type.statsd; + +import com.timgroup.statsd.NonBlockingStatsDClientBuilder; +import com.timgroup.statsd.StatsDClient; +import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +public class DaggerStatsDClient implements SerializedStatsDClientSupplier { + private final int port; + private final String hostname; + private final String[] constantTags; + private static StatsDClient statsDClient; + private static final String PREFIX = "statsd"; + private static final Logger LOGGER = LoggerFactory.getLogger(DaggerStatsDClient.class.getName()); + + public DaggerStatsDClient(String hostname, int port, StatsDTag[] globalTags) { + this.hostname = hostname; + this.port = port; + this.constantTags = Arrays.stream(globalTags) + .map(StatsDTag::getFormattedTag) + .toArray(String[]::new); + } + + @Override + public StatsDClient getClient() { + if (statsDClient == null) { + statsDClient = new NonBlockingStatsDClientBuilder() + .prefix(PREFIX) + .hostname(hostname) + .port(port) + .constantTags(constantTags) + .build(); + String message = String.format("Created new instance of StatsDClient for host=%s and port=%d", hostname, port); + LOGGER.info(message); + } + return statsDClient; + } +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/SerializedStatsDClientSupplier.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/SerializedStatsDClientSupplier.java new file mode 100644 index 000000000..a1f51d061 --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/SerializedStatsDClientSupplier.java @@ -0,0 +1,10 @@ +package io.odpf.dagger.common.metrics.type.statsd; + +import com.timgroup.statsd.StatsDClient; + +import java.io.Serializable; + +@FunctionalInterface +public interface SerializedStatsDClientSupplier extends Serializable { + StatsDClient getClient(); +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerCounterManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerCounterManager.java new file mode 100644 index 000000000..518c40d5c --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerCounterManager.java @@ -0,0 +1,43 @@ +package io.odpf.dagger.common.metrics.type.statsd.manager; + +import com.timgroup.statsd.StatsDClient; +import io.odpf.dagger.common.metrics.aspects.Aspects; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; +import io.odpf.dagger.common.metrics.type.Counter; +import io.odpf.dagger.common.metrics.type.MeasurementManager; + +import java.util.ArrayList; + +public class DaggerCounterManager implements MeasurementManager, Counter { + private final StatsDClient statsDClient; + private String[] formattedTags; + + public DaggerCounterManager(SerializedStatsDClientSupplier statsDClientSupplier) { + this.statsDClient = statsDClientSupplier.getClient(); + } + + @Override + public void register(Aspects[] aspect, StatsDTag[] tags) { + register(tags); + } + + @Override + public void register(StatsDTag[] tags) { + ArrayList tagList = new ArrayList<>(); + for (StatsDTag measurementTag : tags) { + tagList.add(measurementTag.getFormattedTag()); + } + this.formattedTags = tagList.toArray(new String[0]); + } + + @Override + public void increment(Aspects aspect) { + statsDClient.increment(aspect.getValue(), formattedTags); + } + + @Override + public void decrement(Aspects aspect) { + statsDClient.decrement(aspect.getValue(), formattedTags); + } +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerGaugeManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerGaugeManager.java new file mode 100644 index 000000000..2e76d5c93 --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerGaugeManager.java @@ -0,0 +1,48 @@ +package io.odpf.dagger.common.metrics.type.statsd.manager; + +import com.timgroup.statsd.StatsDClient; +import io.odpf.dagger.common.metrics.aspects.Aspects; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; +import io.odpf.dagger.common.metrics.type.Gauge; +import io.odpf.dagger.common.metrics.type.MeasurementManager; + +import java.util.ArrayList; + +public class DaggerGaugeManager implements MeasurementManager, Gauge { + private final StatsDClient statsDClient; + private String[] formattedTags; + + public DaggerGaugeManager(SerializedStatsDClientSupplier statsDClientSupplier) { + this.statsDClient = statsDClientSupplier.getClient(); + } + + @Override + public void register(Aspects[] aspect, StatsDTag[] tags) { + register(tags); + } + + @Override + public void register(StatsDTag[] tags) { + ArrayList tagList = new ArrayList<>(); + for (StatsDTag measurementTag : tags) { + tagList.add(measurementTag.getFormattedTag()); + } + this.formattedTags = tagList.toArray(new String[0]); + } + + @Override + public void registerLong(Aspects aspect, long gaugeValue) { + statsDClient.gauge(aspect.getValue(), gaugeValue, formattedTags); + } + + @Override + public void registerString(Aspects aspect, String gaugeValue) { + throw new UnsupportedOperationException("DogStatsD library doesn't support reporting non-numeric values as gauge values."); + } + + @Override + public void registerDouble(Aspects aspect, double gaugeValue) { + statsDClient.gauge(aspect.getValue(), gaugeValue, formattedTags); + } +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java new file mode 100644 index 000000000..cc71838d8 --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java @@ -0,0 +1,44 @@ +package io.odpf.dagger.common.metrics.type.statsd.manager; + +import com.timgroup.statsd.StatsDClient; +import io.odpf.dagger.common.metrics.aspects.Aspects; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; +import io.odpf.dagger.common.metrics.type.Histogram; +import io.odpf.dagger.common.metrics.type.MeasurementManager; + +import java.util.ArrayList; +import java.util.function.Supplier; + +public class DaggerHistogramManager implements MeasurementManager, Histogram { + private final StatsDClient statsDClient; + private String[] formattedTags; + + public DaggerHistogramManager(SerializedStatsDClientSupplier statsDClientSupplier) { + this.statsDClient = statsDClientSupplier.getClient(); + } + + @Override + public void register(Aspects[] aspect, StatsDTag[] tags) { + register(tags); + } + + @Override + public void register(StatsDTag[] tags) { + ArrayList tagList = new ArrayList<>(); + for (StatsDTag measurementTag : tags) { + tagList.add(measurementTag.getFormattedTag()); + } + this.formattedTags = tagList.toArray(new String[0]); + } + + @Override + public void recordValue(Aspects aspect, long value) { + statsDClient.histogram(aspect.getValue(), value, formattedTags); + } + + @Override + public void recordValue(Aspects aspect, double value) { + statsDClient.histogram(aspect.getValue(), value, formattedTags); + } +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerMeterManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerMeterManager.java new file mode 100644 index 000000000..8bb008868 --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerMeterManager.java @@ -0,0 +1,44 @@ +package io.odpf.dagger.common.metrics.type.statsd.manager; + +import com.timgroup.statsd.StatsDClient; +import io.odpf.dagger.common.metrics.aspects.Aspects; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; +import io.odpf.dagger.common.metrics.type.MeasurementManager; +import io.odpf.dagger.common.metrics.type.Meter; + +import java.util.ArrayList; +import java.util.function.Supplier; + +public class DaggerMeterManager implements MeasurementManager, Meter { + private final StatsDClient statsDClient; + private String[] formattedTags; + + public DaggerMeterManager(SerializedStatsDClientSupplier statsDClientSupplier) { + this.statsDClient = statsDClientSupplier.getClient(); + } + + @Override + public void register(Aspects[] aspect, StatsDTag[] tags) { + register(tags); + } + + @Override + public void register(StatsDTag[] tags) { + ArrayList tagList = new ArrayList<>(); + for (StatsDTag measurementTag : tags) { + tagList.add(measurementTag.getFormattedTag()); + } + this.formattedTags = tagList.toArray(new String[0]); + } + + @Override + public void markEvent(Aspects aspect) { + statsDClient.increment(aspect.getValue(), formattedTags); + } + + @Override + public void markEvent(Aspects aspect, long num) { + statsDClient.count(aspect.getValue(), num, formattedTags); + } +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/tags/StatsDTag.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/tags/StatsDTag.java new file mode 100644 index 000000000..f4ea0c0e1 --- /dev/null +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/tags/StatsDTag.java @@ -0,0 +1,34 @@ +package io.odpf.dagger.common.metrics.type.statsd.tags; + +import org.apache.flink.util.Preconditions; + +public class StatsDTag { + private final String tagKey; + private final String tagValue; + private static final String NIL_TAG_VALUE = "NIL_TAG_VALUE"; + + public StatsDTag(String key, String value) { + validate(key, value); + this.tagKey = key; + this.tagValue = value; + } + + private void validate(String key, String value) { + Preconditions.checkArgument(key != null && !key.isEmpty(), + "Tag key cannot be null or empty"); + Preconditions.checkArgument(value != null && !value.isEmpty(), + "Tag value cannot be null or empty. Refer other constructors for creating a tag with only key"); + } + + public StatsDTag(String tagName) { + this(tagName, NIL_TAG_VALUE); + } + + public String getFormattedTag() { + if (tagValue.equals(NIL_TAG_VALUE)) { + return tagKey; + } else { + return String.format("%s:%s", tagKey, tagValue); + } + } +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java index e502c975c..45d2a78e1 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java @@ -1,5 +1,7 @@ package io.odpf.dagger.core; +import io.odpf.dagger.common.metrics.type.statsd.DaggerStatsDClient; +import io.odpf.dagger.core.metrics.telemetry.statsd.DaggerStatsDClientProvider; import io.odpf.dagger.core.source.Stream; import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.datastream.DataStream; @@ -220,6 +222,8 @@ private void addSink(StreamInfo streamInfo) { } List getStreams() { - return StreamsFactory.getStreams(configuration, stencilClientOrchestrator); + org.apache.flink.configuration.Configuration flinkConfiguration = (org.apache.flink.configuration.Configuration) this.executionEnvironment.getConfiguration(); + DaggerStatsDClient daggerStatsDClient = DaggerStatsDClientProvider.provide(flinkConfiguration, configuration); + return StreamsFactory.getStreams(configuration, stencilClientOrchestrator, daggerStatsDClient); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java new file mode 100644 index 000000000..b44a32fa1 --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java @@ -0,0 +1,32 @@ +package io.odpf.dagger.core.metrics.aspects; + +import io.odpf.dagger.common.metrics.aspects.AspectType; +import io.odpf.dagger.common.metrics.aspects.Aspects; + +public enum ParquetReaderAspects implements Aspects { + READER_CREATED("reader_created", AspectType.Counter), + READER_CLOSED("reader_closed", AspectType.Counter), + READER_ROWS_EMITTED("reader_rows_emitted", AspectType.Counter), + READER_ROW_DESERIALIZATION_TIME("reader_row_deserialization_time", AspectType.Histogram), + READER_ROW_READ_TIME("reader_row_read_time", AspectType.Histogram), + READER_CREATION_EXCEPTION("reader_creation_exception", AspectType.Counter), + READER_DESERIALIZATION_EXCEPTION("reader_deserialization_exception", AspectType.Counter); + + private final String value; + private final AspectType aspectType; + + ParquetReaderAspects(String value, AspectType aspectType) { + this.value = value; + this.aspectType = aspectType; + } + + @Override + public String getValue() { + return value; + } + + @Override + public AspectType getAspectType() { + return aspectType; + } +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/ComponentTags.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/ComponentTags.java new file mode 100644 index 000000000..16d357535 --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/ComponentTags.java @@ -0,0 +1,12 @@ +package io.odpf.dagger.core.metrics.telemetry.statsd; + +import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; + +public class ComponentTags { + public static StatsDTag[] getParquetReaderTags() { + return new StatsDTag[]{ + new StatsDTag("component", "parquet_reader"), + new StatsDTag("dagger_source", "parquet_source") + }; + } +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java new file mode 100644 index 000000000..6a6b8c531 --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java @@ -0,0 +1,41 @@ +package io.odpf.dagger.core.metrics.telemetry.statsd; + +import io.odpf.dagger.common.metrics.type.statsd.DaggerStatsDClient; +import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.Configuration; + +import static io.odpf.dagger.core.utils.Constants.FLINK_JOB_ID_DEFAULT; +import static io.odpf.dagger.core.utils.Constants.FLINK_JOB_ID_KEY; + +public class DaggerStatsDClientProvider { + + private static final String FLINK_STATSD_HOST_CONFIG_KEY = "metrics.reporter.stsd.host"; + private static final String DEFAULT_STATSD_HOST_VALUE = "localhost"; + private static final String FLINK_STATSD_PORT_CONFIG_KEY = "metrics.reporter.stsd.port"; + private static final int DEFAULT_STATSD_PORT_VALUE = 8125; + + public static DaggerStatsDClient provide(Configuration flinkConfiguration, io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { + ConfigOption hostConfigOption = ConfigOptions + .key(FLINK_STATSD_HOST_CONFIG_KEY) + .stringType() + .defaultValue(DEFAULT_STATSD_HOST_VALUE); + ConfigOption portConfigOption = ConfigOptions + .key(FLINK_STATSD_PORT_CONFIG_KEY) + .intType() + .defaultValue(DEFAULT_STATSD_PORT_VALUE); + + String hostname = flinkConfiguration.getString(hostConfigOption); + int port = flinkConfiguration.getInteger(portConfigOption); + StatsDTag[] globalTags = generateGlobalTags(daggerConfiguration); + + return new DaggerStatsDClient(hostname, port, globalTags); + } + + public static StatsDTag[] generateGlobalTags(io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { + return new StatsDTag[]{ + new StatsDTag(GlobalTags.JOB_ID, daggerConfiguration.getString(FLINK_JOB_ID_KEY, FLINK_JOB_ID_DEFAULT)) + }; + } +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/GlobalTags.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/GlobalTags.java new file mode 100644 index 000000000..bdb5703d8 --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/GlobalTags.java @@ -0,0 +1,5 @@ +package io.odpf.dagger.core.metrics.telemetry.statsd; + +public class GlobalTags { + public static final String JOB_ID = "job_id"; +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java index 572a8a758..63e40c319 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java @@ -1,6 +1,7 @@ package io.odpf.dagger.core.source; import io.odpf.dagger.common.configuration.Configuration; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.core.exception.InvalidDaggerSourceException; import io.odpf.dagger.core.source.config.StreamConfig; @@ -16,8 +17,8 @@ public class DaggerSourceFactory { - public static DaggerSource create(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer) { - List> daggerSources = getDaggerSources(streamConfig, configuration, deserializer); + public static DaggerSource create(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDClientSupplier statsDClientSupplier) { + List> daggerSources = getDaggerSources(streamConfig, configuration, deserializer, statsDClientSupplier); return daggerSources.stream() .filter(DaggerSource::canBuild) .findFirst() @@ -28,10 +29,10 @@ public static DaggerSource create(StreamConfig streamConfig, Configuration }); } - private static List> getDaggerSources(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer) { + private static List> getDaggerSources(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDClientSupplier statsDClientSupplier) { KafkaDaggerSource kafkaDaggerSource = new KafkaDaggerSource(streamConfig, configuration, deserializer); FlinkKafkaConsumerDaggerSource flinkKafkaConsumerDaggerSource = new FlinkKafkaConsumerDaggerSource(streamConfig, configuration, deserializer); - ParquetDaggerSource parquetDaggerSource = new ParquetDaggerSource(streamConfig, configuration, deserializer); + ParquetDaggerSource parquetDaggerSource = new ParquetDaggerSource(streamConfig, configuration, deserializer, statsDClientSupplier); return Stream.of(kafkaDaggerSource, flinkKafkaConsumerDaggerSource, parquetDaggerSource) .collect(Collectors.toList()); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java index 6a03aa0b5..cbf3e3e7c 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java @@ -2,6 +2,7 @@ import io.odpf.dagger.common.configuration.Configuration; import io.odpf.dagger.common.core.StencilClientOrchestrator; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.core.deserializer.DaggerDeserializerFactory; import io.odpf.dagger.core.source.config.StreamConfig; @@ -32,16 +33,18 @@ public static class Builder { private final StreamConfig streamConfig; private final Configuration configuration; private final StencilClientOrchestrator stencilClientOrchestrator; + private final SerializedStatsDClientSupplier statsDClientSupplier; - public Builder(StreamConfig streamConfig, Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator) { + public Builder(StreamConfig streamConfig, Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator, SerializedStatsDClientSupplier statsDClientSupplier) { this.streamConfig = streamConfig; this.configuration = configuration; this.stencilClientOrchestrator = stencilClientOrchestrator; + this.statsDClientSupplier = statsDClientSupplier; } public Stream build() { DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator); - DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, daggerDeserializer); + DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, daggerDeserializer, statsDClientSupplier); return new Stream(daggerSource, streamConfig.getSchemaTable()); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/StreamsFactory.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/StreamsFactory.java index 3ec25a7e3..de3886e57 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/StreamsFactory.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/StreamsFactory.java @@ -2,18 +2,19 @@ import io.odpf.dagger.common.configuration.Configuration; import io.odpf.dagger.common.core.StencilClientOrchestrator; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.core.source.config.StreamConfig; import java.util.ArrayList; import java.util.List; public class StreamsFactory { - public static List getStreams(Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator) { + public static List getStreams(Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator, SerializedStatsDClientSupplier statsDClientSupplier) { StreamConfig[] streamConfigs = StreamConfig.parse(configuration); ArrayList streams = new ArrayList<>(); for (StreamConfig streamConfig : streamConfigs) { - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplier); streams.add(builder.build()); } return streams; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java index 781803e79..499ae63f4 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java @@ -1,6 +1,7 @@ package io.odpf.dagger.core.source.parquet; import io.odpf.dagger.common.configuration.Configuration; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.DaggerConfigurationException; @@ -33,13 +34,15 @@ public class ParquetDaggerSource implements DaggerSource { private final DaggerDeserializer deserializer; private final StreamConfig streamConfig; private final Configuration configuration; + private final SerializedStatsDClientSupplier statsDClientSupplier; private static final SourceType SUPPORTED_SOURCE_TYPE = BOUNDED; private static final SourceName SUPPORTED_SOURCE_NAME = PARQUET_SOURCE; - public ParquetDaggerSource(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer) { + public ParquetDaggerSource(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDClientSupplier statsDClientSupplier) { this.streamConfig = streamConfig; this.configuration = configuration; this.deserializer = deserializer; + this.statsDClientSupplier = statsDClientSupplier; } @Override @@ -99,12 +102,13 @@ private FileSplitAssigner.Provider buildParquetFileSplitAssignerProvider() { private ParquetFileRecordFormat buildParquetFileRecordFormat() { SimpleGroupDeserializer simpleGroupDeserializer = (SimpleGroupDeserializer) deserializer; - ReaderProvider parquetFileReaderProvider = new ParquetReader.ParquetReaderProvider(simpleGroupDeserializer); + ReaderProvider parquetFileReaderProvider = new ParquetReader.ParquetReaderProvider(simpleGroupDeserializer, statsDClientSupplier); ParquetFileRecordFormat.Builder parquetFileRecordFormatBuilder = ParquetFileRecordFormat.Builder.getInstance(); Supplier> typeInformationProvider = (Supplier> & Serializable) simpleGroupDeserializer::getProducedType; return parquetFileRecordFormatBuilder .setParquetFileReaderProvider(parquetFileReaderProvider) .setTypeInformationProvider(typeInformationProvider) + .setStatsDClientSupplier(statsDClientSupplier) .build(); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java index 9828dfabb..a9775c545 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java @@ -2,6 +2,7 @@ import static com.google.api.client.util.Preconditions.checkArgument; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.core.source.parquet.reader.ReaderProvider; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; @@ -14,10 +15,13 @@ public class ParquetFileRecordFormat implements FileRecordFormat { private final ReaderProvider parquetFileReaderProvider; private final Supplier> typeInformationProvider; + private final SerializedStatsDClientSupplier statsDClientSupplier; - private ParquetFileRecordFormat(ReaderProvider parquetFileReaderProvider, Supplier> typeInformationProvider) { + private ParquetFileRecordFormat(ReaderProvider parquetFileReaderProvider, Supplier> + typeInformationProvider, SerializedStatsDClientSupplier statsDClientSupplier) { this.parquetFileReaderProvider = parquetFileReaderProvider; this.typeInformationProvider = typeInformationProvider; + this.statsDClientSupplier = statsDClientSupplier; } @Override @@ -44,6 +48,7 @@ public TypeInformation getProducedType() { public static class Builder { private ReaderProvider parquetFileReaderProvider; private Supplier> typeInformationProvider; + private SerializedStatsDClientSupplier statsDClientSupplier; public static Builder getInstance() { return new Builder(); @@ -64,10 +69,16 @@ public Builder setTypeInformationProvider(Supplier> typeInf return this; } + public Builder setStatsDClientSupplier(SerializedStatsDClientSupplier statsDClientSupplier) { + this.statsDClientSupplier = statsDClientSupplier; + return this; + } + public ParquetFileRecordFormat build() { checkArgument(parquetFileReaderProvider != null, "ReaderProvider is required but is set as null"); checkArgument(typeInformationProvider != null, "TypeInformationProvider is required but is set as null"); - return new ParquetFileRecordFormat(parquetFileReaderProvider, typeInformationProvider); + checkArgument(statsDClientSupplier != null, "StatsDClientSupplier object is required but is set as null"); + return new ParquetFileRecordFormat(parquetFileReaderProvider, typeInformationProvider, statsDClientSupplier); } } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java index 573c1b859..411548778 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java @@ -1,7 +1,12 @@ package io.odpf.dagger.core.source.parquet.reader; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.common.metrics.type.statsd.manager.DaggerCounterManager; +import io.odpf.dagger.common.metrics.type.statsd.manager.DaggerHistogramManager; +import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.ParquetFileSourceReaderInitializationException; +import io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects; import org.apache.flink.connector.file.src.reader.FileRecordFormat; import org.apache.flink.connector.file.src.util.CheckpointedPosition; import org.apache.flink.types.Row; @@ -22,6 +27,9 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.time.Instant; + +import static io.odpf.dagger.core.metrics.telemetry.statsd.ComponentTags.getParquetReaderTags; public class ParquetReader implements FileRecordFormat.Reader { private final Path hadoopFilePath; @@ -33,15 +41,28 @@ public class ParquetReader implements FileRecordFormat.Reader { private RecordReader recordReader; private final MessageType schema; private long totalEmittedRowCount; + private DaggerCounterManager daggerCounterManager; + private DaggerHistogramManager daggerHistogramManager; private static final Logger LOGGER = LoggerFactory.getLogger(ParquetReader.class.getName()); - private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDeserializer, ParquetFileReader parquetFileReader) throws IOException { + private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDeserializer, ParquetFileReader + parquetFileReader, SerializedStatsDClientSupplier statsDClientSupplier) throws IOException { this.hadoopFilePath = hadoopFilePath; this.simpleGroupDeserializer = simpleGroupDeserializer; this.parquetFileReader = parquetFileReader; this.schema = this.parquetFileReader.getFileMetaData().getSchema(); this.isRecordReaderInitialized = false; this.totalEmittedRowCount = 0L; + this.registerTagsWithMeasurementManagers(statsDClientSupplier); + daggerCounterManager.increment(ParquetReaderAspects.READER_CREATED); + } + + private void registerTagsWithMeasurementManagers(SerializedStatsDClientSupplier statsDClientSupplier) { + StatsDTag[] parquetReaderTags = getParquetReaderTags(); + this.daggerCounterManager = new DaggerCounterManager(statsDClientSupplier); + this.daggerCounterManager.register(parquetReaderTags); + this.daggerHistogramManager = new DaggerHistogramManager(statsDClientSupplier); + this.daggerHistogramManager.register(parquetReaderTags); } private boolean checkIfNullPage(PageReadStore page) { @@ -69,6 +90,8 @@ private void initializeRecordReader() throws IOException { } private Row readRecords() throws IOException { + long startReadTime = Instant.now().toEpochMilli(); + if (currentRecordIndex >= rowCount) { PageReadStore nextPage = parquetFileReader.readNextRowGroup(); if (checkIfNullPage(nextPage)) { @@ -76,7 +99,18 @@ private Row readRecords() throws IOException { } changeReaderPosition(nextPage); } - return readAndDeserialize(); + SimpleGroup simpleGroup = (SimpleGroup) recordReader.read(); + long endReadTime = Instant.now().toEpochMilli(); + + currentRecordIndex++; + long startDeserializationTime = Instant.now().toEpochMilli(); + Row row = simpleGroupDeserializer.deserialize(simpleGroup); + long endDeserializationTime = Instant.now().toEpochMilli(); + totalEmittedRowCount++; + + daggerHistogramManager.recordValue(ParquetReaderAspects.READER_ROW_READ_TIME, endReadTime - startReadTime); + daggerHistogramManager.recordValue(ParquetReaderAspects.READER_ROW_DESERIALIZATION_TIME, endDeserializationTime - startDeserializationTime); + return row; } @Nullable @@ -85,14 +119,8 @@ public Row read() throws IOException { if (!isRecordReaderInitialized) { initializeRecordReader(); } - return readRecords(); - } - - private Row readAndDeserialize() { - SimpleGroup simpleGroup = (SimpleGroup) recordReader.read(); - currentRecordIndex++; - Row row = simpleGroupDeserializer.deserialize(simpleGroup); - totalEmittedRowCount++; + Row row = readRecords(); + daggerCounterManager.increment(ParquetReaderAspects.READER_ROWS_EMITTED); return row; } @@ -102,6 +130,7 @@ public void close() throws IOException { closeRecordReader(); String logMessage = String.format("Closed the ParquetFileReader and de-referenced the RecordReader for file %s", hadoopFilePath.getName()); LOGGER.info(logMessage); + daggerCounterManager.increment(ParquetReaderAspects.READER_CLOSED); } private void closeRecordReader() { @@ -118,9 +147,11 @@ public CheckpointedPosition getCheckpointedPosition() { public static class ParquetReaderProvider implements ReaderProvider { private final SimpleGroupDeserializer simpleGroupDeserializer; + private final SerializedStatsDClientSupplier statsDClientSupplier; - public ParquetReaderProvider(SimpleGroupDeserializer simpleGroupDeserializer) { + public ParquetReaderProvider(SimpleGroupDeserializer simpleGroupDeserializer, SerializedStatsDClientSupplier statsDClientSupplier) { this.simpleGroupDeserializer = simpleGroupDeserializer; + this.statsDClientSupplier = statsDClientSupplier; } @Override @@ -129,7 +160,7 @@ public ParquetReader getReader(String filePath) { Configuration conf = new Configuration(); Path hadoopFilePath = new Path(filePath); ParquetFileReader parquetFileReader = ParquetFileReader.open(HadoopInputFile.fromPath(hadoopFilePath, conf)); - return new ParquetReader(hadoopFilePath, simpleGroupDeserializer, parquetFileReader); + return new ParquetReader(hadoopFilePath, simpleGroupDeserializer, parquetFileReader, statsDClientSupplier); } catch (IOException | RuntimeException ex) { throw new ParquetFileSourceReaderInitializationException(ex); } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java index b94b8d8c3..8bf0ebc73 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java @@ -1,6 +1,8 @@ package io.odpf.dagger.core.source; +import com.timgroup.statsd.StatsDClient; import io.odpf.dagger.common.configuration.Configuration; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.common.serde.json.deserialization.JsonDeserializer; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.common.serde.proto.deserialization.ProtoDeserializer; @@ -19,6 +21,7 @@ import org.mockito.Mockito; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -29,6 +32,8 @@ public class DaggerSourceFactoryTest { @Mock private Configuration configuration; + private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + @Before public void setUp() throws Exception { initMocks(this); @@ -38,7 +43,7 @@ public void setUp() throws Exception { public void shouldReturnKafkaDaggerSourceWhenConfigured() { ProtoDeserializer deserializer = Mockito.mock(ProtoDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.KAFKA_SOURCE, SourceType.UNBOUNDED)}); - DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer); + DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDClientSupplierMock); assertTrue(daggerSource instanceof KafkaDaggerSource); } @@ -47,7 +52,7 @@ public void shouldReturnKafkaDaggerSourceWhenConfigured() { public void shouldReturnFlinkKafkaConsumerDaggerSourceWhenConfigured() { JsonDeserializer deserializer = Mockito.mock(JsonDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.KAFKA_CONSUMER, SourceType.UNBOUNDED)}); - DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer); + DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDClientSupplierMock); assertTrue(daggerSource instanceof FlinkKafkaConsumerDaggerSource); } @@ -56,7 +61,7 @@ public void shouldReturnFlinkKafkaConsumerDaggerSourceWhenConfigured() { public void shouldReturnParquetDaggerSourceWhenConfigured() { SimpleGroupDeserializer deserializer = Mockito.mock(SimpleGroupDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED)}); - DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer); + DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDClientSupplierMock); assertTrue(daggerSource instanceof ParquetDaggerSource); } @@ -66,6 +71,6 @@ public void shouldThrowRuntimeExceptionIfNoDaggerSourceCouldBeCreatedAsPerConfig SimpleGroupDeserializer deserializer = Mockito.mock(SimpleGroupDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.UNBOUNDED)}); - assertThrows(InvalidDaggerSourceException.class, () -> DaggerSourceFactory.create(streamConfig, configuration, deserializer)); + assertThrows(InvalidDaggerSourceException.class, () -> DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDClientSupplierMock)); } } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamTest.java index 828129d99..78d120195 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamTest.java @@ -1,7 +1,9 @@ package io.odpf.dagger.core.source; +import com.timgroup.statsd.StatsDClient; import io.odpf.dagger.common.configuration.Configuration; import io.odpf.dagger.common.core.StencilClientOrchestrator; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.source.config.StreamConfig; import io.odpf.dagger.core.source.config.models.SourceDetails; @@ -25,6 +27,7 @@ import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -53,6 +56,8 @@ public class StreamTest { @Mock private DaggerSource mockDaggerSource; + private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + @Before public void setup() { initMocks(this); @@ -68,7 +73,7 @@ public void shouldBeAbleToBuildAStreamWithKafkaDaggerSourceAndProtoSchema() { when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof KafkaDaggerSource); @@ -93,7 +98,7 @@ public void shouldBeAbleToBuildAStreamWithFlinkKafkaConsumerDaggerSourceAndProto when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof FlinkKafkaConsumerDaggerSource); @@ -106,7 +111,7 @@ public void shouldBeAbleToBuildAStreamWithKafkaDaggerSourceAndJsonSchema() { when(streamConfig.getSchemaTable()).thenReturn("data_stream"); when(streamConfig.getJsonSchema()).thenReturn("{ \"$schema\": \"https://json-schema.org/draft/2020-12/schema\", \"$id\": \"https://example.com/product.schema.json\", \"title\": \"Product\", \"description\": \"A product from Acme's catalog\", \"type\": \"object\", \"properties\": { \"id\": { \"description\": \"The unique identifier for a product\", \"type\": \"string\" }, \"time\": { \"description\": \"event timestamp of the event\", \"type\": \"string\", \"format\" : \"date-time\" } }, \"required\": [ \"id\", \"time\" ] }"); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof KafkaDaggerSource); @@ -119,7 +124,7 @@ public void shouldBeAbleToBuildAStreamWithFlinkKafkaConsumerDaggerSourceAndJsonS when(streamConfig.getSchemaTable()).thenReturn("data_stream"); when(streamConfig.getJsonSchema()).thenReturn("{ \"$schema\": \"https://json-schema.org/draft/2020-12/schema\", \"$id\": \"https://example.com/product.schema.json\", \"title\": \"Product\", \"description\": \"A product from Acme's catalog\", \"type\": \"object\", \"properties\": { \"id\": { \"description\": \"The unique identifier for a product\", \"type\": \"string\" }, \"time\": { \"description\": \"event timestamp of the event\", \"type\": \"string\", \"format\" : \"date-time\" } }, \"required\": [ \"id\", \"time\" ] }"); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof FlinkKafkaConsumerDaggerSource); @@ -134,7 +139,7 @@ public void shouldBeAbleToBuildAStreamWithParquetDaggerSourceAndProtoSchema() { when(streamConfig.getSchemaTable()).thenReturn("data_stream"); when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof ParquetDaggerSource); diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamsFactoryTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamsFactoryTest.java index c5f2de52f..dbc7be556 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamsFactoryTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamsFactoryTest.java @@ -1,5 +1,7 @@ package io.odpf.dagger.core.source; +import com.timgroup.statsd.StatsDClient; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.core.source.flinkkafkaconsumer.FlinkKafkaConsumerDaggerSource; import io.odpf.dagger.core.source.kafka.KafkaDaggerSource; import io.odpf.stencil.client.StencilClient; @@ -18,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -31,6 +34,8 @@ public class StreamsFactoryTest { @Mock private Configuration configuration; + private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + @Before public void setup() { initMocks(this); @@ -65,7 +70,7 @@ public void shouldReturnListOfStreamsCreatedFromConfiguration() { when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - List streams = StreamsFactory.getStreams(configuration, stencilClientOrchestrator); + List streams = StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDClientSupplierMock); assertEquals(2, streams.size()); assertTrue(streams.get(0).getDaggerSource() instanceof FlinkKafkaConsumerDaggerSource); @@ -88,7 +93,7 @@ public void shouldThrowErrorForInvalidStreamConfig() { when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); assertThrows(JsonSyntaxException.class, - () -> StreamsFactory.getStreams(configuration, stencilClientOrchestrator)); + () -> StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDClientSupplierMock)); } @Test @@ -96,6 +101,6 @@ public void shouldThrowNullPointerIfStreamConfigIsNotGiven() { when(configuration.getString(INPUT_STREAMS, "")).thenReturn(""); assertThrows(NullPointerException.class, - () -> StreamsFactory.getStreams(configuration, stencilClientOrchestrator)); + () -> StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDClientSupplierMock)); } } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java index 744134124..98476292f 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java @@ -1,6 +1,8 @@ package io.odpf.dagger.core.source.parquet; +import com.timgroup.statsd.StatsDClient; import io.odpf.dagger.common.configuration.Configuration; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.common.serde.proto.deserialization.ProtoDeserializer; @@ -46,6 +48,8 @@ public class ParquetDaggerSourceTest { @Mock private StreamExecutionEnvironment streamExecutionEnvironment; + private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + private FileSource fileSource; @Before @@ -59,7 +63,7 @@ public void setup() { @Test public void shouldBeAbleToBuildSourceIfSourceDetailsIsBoundedParquetAndDaggerDeserializerIsSimpleGroupDeserializer() { when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); assertTrue(daggerSource.canBuild()); } @@ -68,7 +72,7 @@ public void shouldBeAbleToBuildSourceIfSourceDetailsIsBoundedParquetAndDaggerDes public void shouldNotBeAbleToBuildSourceIfSourceDetailsContainsMultipleBackToBackSources() { when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED), new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); assertFalse(daggerSource.canBuild()); } @@ -76,7 +80,7 @@ public void shouldNotBeAbleToBuildSourceIfSourceDetailsContainsMultipleBackToBac @Test public void shouldNotBeAbleToBuildSourceIfSourceNameIsUnsupported() { when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.KAFKA_CONSUMER, SourceType.BOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); assertFalse(daggerSource.canBuild()); } @@ -84,7 +88,7 @@ public void shouldNotBeAbleToBuildSourceIfSourceNameIsUnsupported() { @Test public void shouldNotBeAbleToBuildSourceIfSourceTypeIsUnsupported() { when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.UNBOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); assertFalse(daggerSource.canBuild()); } @@ -93,14 +97,14 @@ public void shouldNotBeAbleToBuildSourceIfSourceTypeIsUnsupported() { public void shouldNotBeAbleToBuildSourceIfDeserializerTypeIsUnsupported() { DaggerDeserializer unsupportedDeserializer = Mockito.mock(ProtoDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, unsupportedDeserializer); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, unsupportedDeserializer, statsDClientSupplierMock); assertFalse(daggerSource.canBuild()); } @Test public void shouldBeAbleToRegisterSourceWithExecutionEnvironmentForCorrectConfiguration() { - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); ParquetDaggerSource daggerSourceSpy = Mockito.spy(daggerSource); doReturn(fileSource).when(daggerSourceSpy).buildFileSource(); when(streamConfig.getSchemaTable()).thenReturn("data_stream_0"); @@ -117,7 +121,7 @@ public void shouldUseStreamConfigurationToBuildTheFileSource() { when(streamConfig.getParquetFilesReadOrderStrategy()).thenReturn(EARLIEST_TIME_URL_FIRST); when(streamConfig.getParquetFilePaths()).thenReturn(new String[]{"gs://sshsh", "gs://shadd"}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); daggerSource.register(streamExecutionEnvironment, strategy); } @@ -127,7 +131,7 @@ public void shouldThrowRuntimeExceptionIfReadOrderStrategyIsNotSupported() { when(streamConfig.getParquetFilesReadOrderStrategy()).thenReturn(EARLIEST_INDEX_FIRST); when(streamConfig.getParquetFilePaths()).thenReturn(new String[]{"gs://sshsh", "gs://shadd"}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); assertThrows(DaggerConfigurationException.class, () -> daggerSource.register(streamExecutionEnvironment, strategy)); } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java index f87c87a28..6a8bb4f32 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java @@ -1,6 +1,8 @@ package io.odpf.dagger.core.source.parquet; +import com.timgroup.statsd.StatsDClient; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.core.source.parquet.reader.ParquetReader; import io.odpf.dagger.core.source.parquet.reader.ReaderProvider; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -15,6 +17,7 @@ import java.util.function.Supplier; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; import static org.mockito.MockitoAnnotations.initMocks; public class ParquetFileRecordFormatTest { @@ -30,6 +33,7 @@ public class ParquetFileRecordFormatTest { private final ReaderProvider readerProviderMock = (filePath) -> parquetReader; private final Supplier> typeInformationProviderMock = () -> typeInformation; + private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); @Before public void setup() { @@ -41,6 +45,7 @@ public void shouldBuildAFileRecordFormatAsPerConfiguredParameters() { ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setParquetFileReaderProvider(readerProviderMock) .setTypeInformationProvider(typeInformationProviderMock) + .setStatsDClientSupplier(statsDClientSupplierMock) .build(); FileRecordFormat.Reader expectedReader = parquetFileRecordFormat.createReader(configuration, new Path("gs://file-path"), 0, 1024); @@ -57,6 +62,7 @@ public void shouldThrowIllegalArgumentExceptionWhenReaderProviderIsNotConfigured IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder .setTypeInformationProvider(typeInformationProviderMock) + .setStatsDClientSupplier(statsDClientSupplierMock) .build()); assertEquals("ReaderProvider is required but is set as null", ex.getMessage()); @@ -68,6 +74,7 @@ public void shouldThrowIllegalArgumentExceptionWhenTypeInformationProviderIsNotC IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder .setParquetFileReaderProvider(readerProviderMock) + .setStatsDClientSupplier(statsDClientSupplierMock) .build()); assertEquals("TypeInformationProvider is required but is set as null", ex.getMessage()); @@ -78,6 +85,7 @@ public void shouldReturnFalseWhenIsSplittableIsCalled() { ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setParquetFileReaderProvider(readerProviderMock) .setTypeInformationProvider(typeInformationProviderMock) + .setStatsDClientSupplier(statsDClientSupplierMock) .build(); assertFalse(parquetFileRecordFormat.isSplittable()); @@ -88,6 +96,7 @@ public void shouldThrowUnsupportedOperationExceptionWhenRestoreReaderIsCalled() ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setTypeInformationProvider(typeInformationProviderMock) .setParquetFileReaderProvider(readerProviderMock) + .setStatsDClientSupplier(statsDClientSupplierMock) .build(); UnsupportedOperationException ex = assertThrows(UnsupportedOperationException.class, diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java index 51a9a65c3..4110764d5 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java @@ -1,6 +1,8 @@ package io.odpf.dagger.core.source.parquet.reader; +import com.timgroup.statsd.StatsDClient; +import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.ParquetFileSourceReaderInitializationException; import org.apache.commons.lang3.ArrayUtils; @@ -25,6 +27,7 @@ import static org.apache.parquet.schema.Types.*; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -37,6 +40,8 @@ public class ParquetReaderTest { @Rule public TemporaryFolder tempFolder = TemporaryFolder.builder().assureDeletion().build(); + private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + @Before public void setup() { initMocks(this); @@ -45,7 +50,7 @@ public void setup() { @Test public void shouldCreateReadersConfiguredWithTheSameDeserializerButForDifferentFilePaths() throws IOException { when(deserializer.deserialize(any(SimpleGroup.class))).thenReturn(Row.of("same", "deserializer")); - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); String filePath1 = classLoader.getResource("test_file.parquet").getPath(); @@ -59,7 +64,7 @@ public void shouldCreateReadersConfiguredWithTheSameDeserializerButForDifferentF @Test public void shouldReadFileAndCallDeserializerWithSimpleGroupWhenReadIsCalled() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); @@ -84,7 +89,7 @@ public void shouldReadFileAndCallDeserializerWithSimpleGroupWhenReadIsCalled() t @Test public void shouldBeAbleToReadParquetFileContainingMultipleRowGroups() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("multiple_row_groups_test_file.parquet").getPath()); @@ -112,7 +117,7 @@ public void shouldBeAbleToReadParquetFileContainingMultipleRowGroups() throws IO @Test public void shouldReturnDeserializedValueWhenRecordsPresentAndNullWhenNoMoreDataLeftToRead() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); when(deserializer.deserialize(any(SimpleGroup.class))).thenReturn(Row.of("some value")); @@ -125,7 +130,7 @@ public void shouldReturnDeserializedValueWhenRecordsPresentAndNullWhenNoMoreData @Test public void shouldThrowIOExceptionIfReadIsCalledAfterCallingClose() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); @@ -137,14 +142,14 @@ public void shouldThrowIOExceptionIfReadIsCalledAfterCallingClose() throws IOExc @Test public void shouldThrowParquetFileSourceReaderInitializationExceptionIfCannotConstructReaderForTheFile() throws IOException { final File tempFile = tempFolder.newFile("test_file.parquet"); - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); assertThrows(ParquetFileSourceReaderInitializationException.class, () -> provider.getReader(tempFile.getPath())); } @Test public void shouldReturnCheckPointedPositionWithNoOffsetAndZeroRecordsAfterOffsetWhenReadHasNotBeenCalledYet() { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); @@ -155,7 +160,7 @@ public void shouldReturnCheckPointedPositionWithNoOffsetAndZeroRecordsAfterOffse @Test public void shouldUpdateCheckPointedPositionWithNoOffsetAndCountOfTotalRecordsReadYet() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); @@ -168,7 +173,7 @@ public void shouldUpdateCheckPointedPositionWithNoOffsetAndCountOfTotalRecordsRe @Test public void shouldNotUpdateCheckpointedPositionWhenNoMoreRecordsToRead() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); From f1a5572595de3fd7883c3121de7df702dcbc9fb6 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 28 Jun 2022 13:40:22 +0530 Subject: [PATCH 08/47] feat: cache the global tags for perf reasons - remove unused imports [https://github.com/odpf/dagger/issues/108] --- .../type/statsd/manager/DaggerHistogramManager.java | 1 - .../telemetry/statsd/DaggerStatsDClientProvider.java | 10 +++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java index cc71838d8..2f7c90327 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java @@ -8,7 +8,6 @@ import io.odpf.dagger.common.metrics.type.MeasurementManager; import java.util.ArrayList; -import java.util.function.Supplier; public class DaggerHistogramManager implements MeasurementManager, Histogram { private final StatsDClient statsDClient; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java index 6a6b8c531..fc9fc8ccd 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java @@ -15,6 +15,7 @@ public class DaggerStatsDClientProvider { private static final String DEFAULT_STATSD_HOST_VALUE = "localhost"; private static final String FLINK_STATSD_PORT_CONFIG_KEY = "metrics.reporter.stsd.port"; private static final int DEFAULT_STATSD_PORT_VALUE = 8125; + private static StatsDTag[] globalTags; public static DaggerStatsDClient provide(Configuration flinkConfiguration, io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { ConfigOption hostConfigOption = ConfigOptions @@ -34,8 +35,11 @@ public static DaggerStatsDClient provide(Configuration flinkConfiguration, io.od } public static StatsDTag[] generateGlobalTags(io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { - return new StatsDTag[]{ - new StatsDTag(GlobalTags.JOB_ID, daggerConfiguration.getString(FLINK_JOB_ID_KEY, FLINK_JOB_ID_DEFAULT)) - }; + if (globalTags == null) { + globalTags = new StatsDTag[]{ + new StatsDTag(GlobalTags.JOB_ID, daggerConfiguration.getString(FLINK_JOB_ID_KEY, FLINK_JOB_ID_DEFAULT)), + }; + } + return globalTags; } } From a6cee081790aca476cd72f4277867fff42fc7233 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Wed, 29 Jun 2022 13:53:16 +0530 Subject: [PATCH 09/47] feat: use Depot and remove dogstatsd [https://github.com/odpf/dagger/issues/108] --- dagger-common/build.gradle | 4 +--- dagger-core/build.gradle | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dagger-common/build.gradle b/dagger-common/build.gradle index 62188c414..318450acc 100644 --- a/dagger-common/build.gradle +++ b/dagger-common/build.gradle @@ -66,9 +66,7 @@ dependencies { dependenciesCommonJar 'org.apache.flink:flink-metrics-dropwizard:' + flinkVersion dependenciesCommonJar 'org.apache.flink:flink-json:' + flinkVersion dependenciesCommonJar 'com.jayway.jsonpath:json-path:2.4.0' - dependenciesCommonJar('io.odpf:stencil:0.1.6') { - exclude group: 'com.datadoghq', module: 'java-dogstatsd-client' - } + dependenciesCommonJar 'io.odpf:stencil:0.1.6' dependenciesCommonJar 'com.datadoghq:java-dogstatsd-client:4.0.0' dependenciesCommonJar 'com.google.code.gson:gson:2.8.2' dependenciesCommonJar 'org.apache.parquet:parquet-column:1.12.2' diff --git a/dagger-core/build.gradle b/dagger-core/build.gradle index 4da113eab..6c95e910f 100644 --- a/dagger-core/build.gradle +++ b/dagger-core/build.gradle @@ -71,6 +71,7 @@ dependencies { implementation 'org.apache.flink:flink-table:' + flinkVersion implementation 'org.apache.flink:flink-table-api-java-bridge_2.11:' + flinkVersion implementation 'org.apache.flink:flink-table-planner_2.11:' + flinkVersion + implementation 'io.odpf:depot:0.1.5' dependenciesJar project(path: ':dagger-common', configuration: 'dependenciesCommonJar') dependenciesJar project(path: ':dagger-functions', configuration: 'dependenciesFunctionsJar') From b2ebf92c1a2242da5dfffd6ee73ad3e3529f87ca Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 5 Jul 2022 14:18:00 +0530 Subject: [PATCH 10/47] feat: add depot to minimalJar - use statsDReporter for raising metrics [https://github.com/odpf/dagger/issues/108] --- .../CounterStatsManager.java | 2 +- .../flink => manager}/GaugeStatsManager.java | 2 +- .../flink => manager}/MeterStatsManager.java | 2 +- .../type/statsd/DaggerStatsDClient.java | 41 --------------- .../SerializedStatsDClientSupplier.java | 10 ---- .../statsd/manager/DaggerCounterManager.java | 43 --------------- .../statsd/manager/DaggerMeterManager.java | 44 ---------------- .../odpf/dagger/common/udfs/AggregateUdf.java | 2 +- .../io/odpf/dagger/common/udfs/ScalarUdf.java | 2 +- .../io/odpf/dagger/common/udfs/TableUdf.java | 2 +- .../type/flink/CounterStatsManagerTest.java | 1 + .../type/flink/GaugeStatsManagerTest.java | 1 + .../type/flink/MeterStatsManagerTest.java | 1 + dagger-core/build.gradle | 2 +- .../io/odpf/dagger/core/StreamManager.java | 8 +-- ...ChronologyOrderedSplitAssignerAspects.java | 28 ++++++++++ .../reporters/statsd/DaggerMetricsConfig.java | 43 +++++++++++++++ .../statsd/DaggerStatsDReporterProvider.java | 41 +++++++++++++++ .../SerializedStatsDReporterSupplier.java | 10 ++++ .../statsd/manager/DaggerCounterManager.java | 52 +++++++++++++++++++ .../statsd/manager/DaggerGaugeManager.java | 22 ++++---- .../manager/DaggerHistogramManager.java | 21 ++++---- .../statsd/manager}/MeasurementManager.java | 4 +- .../statsd/measurement}/Counter.java | 6 ++- .../reporters/statsd/measurement}/Gauge.java | 2 +- .../statsd/measurement}/Histogram.java | 2 +- .../reporters/statsd/measurement}/Meter.java | 2 +- .../reporters/statsd/tags/ComponentTags.java | 28 ++++++++++ .../statsd/tags}/GlobalTags.java | 2 +- .../reporters}/statsd/tags/StatsDTag.java | 2 +- .../telemetry/statsd/ComponentTags.java | 12 ----- .../statsd/DaggerStatsDClientProvider.java | 45 ---------------- .../processors/common/EndpointHandler.java | 2 +- .../common/PostResponseTelemetry.java | 2 +- .../processors/external/AsyncConnector.java | 2 +- .../external/es/EsAsyncConnector.java | 2 +- .../external/es/EsResponseHandler.java | 2 +- .../external/grpc/GrpcAsyncConnector.java | 2 +- .../external/grpc/GrpcResponseHandler.java | 2 +- .../external/http/HttpAsyncConnector.java | 2 +- .../external/http/HttpResponseHandler.java | 2 +- .../external/pg/PgAsyncConnector.java | 2 +- .../external/pg/PgResponseHandler.java | 2 +- .../longbow/processor/LongbowReader.java | 2 +- .../longbow/processor/LongbowWriter.java | 2 +- .../processor/MetricsTelemetryExporter.java | 2 +- .../core/source/DaggerSourceFactory.java | 10 ++-- .../io/odpf/dagger/core/source/Stream.java | 10 ++-- .../dagger/core/source/StreamsFactory.java | 6 +-- .../source/parquet/ParquetDaggerSource.java | 12 ++--- .../parquet/ParquetFileRecordFormat.java | 15 +----- .../source/parquet/reader/ParquetReader.java | 28 +++++----- .../ChronologyOrderedSplitAssigner.java | 31 ++++++++++- .../common/EndpointHandlerTest.java | 2 +- .../external/es/EsAsyncConnectorTest.java | 2 +- .../external/es/EsResponseHandlerTest.java | 2 +- .../external/grpc/GrpcAsyncConnectorTest.java | 2 +- .../grpc/GrpcResponseHandlerTest.java | 2 +- .../external/http/HttpAsyncConnectorTest.java | 2 +- .../http/HttpResponseHandlerTest.java | 2 +- .../external/pg/PgAsyncConnectorTest.java | 2 +- .../external/pg/PgResponseHandlerTest.java | 2 +- .../longbow/processor/LongbowReaderTest.java | 2 +- .../longbow/processor/LongbowWriterTest.java | 2 +- .../MetricsTelemetryExporterTest.java | 2 +- .../core/source/DaggerSourceFactoryTest.java | 14 ++--- .../odpf/dagger/core/source/StreamTest.java | 16 +++--- .../core/source/StreamsFactoryTest.java | 12 ++--- .../parquet/ParquetDaggerSourceTest.java | 22 ++++---- .../parquet/ParquetFileRecordFormatTest.java | 9 ---- .../parquet/reader/ParquetReaderTest.java | 24 ++++----- .../InvalidRecordFilterTransformer.java | 2 +- .../functions/udfs/scalar/DartContains.java | 2 +- .../dagger/functions/udfs/scalar/DartGet.java | 2 +- .../udfs/scalar/dart/store/gcs/GcsClient.java | 2 +- .../scalar/dart/store/gcs/GcsDataStore.java | 4 +- .../udfs/scalar/DartContainsTest.java | 2 +- .../functions/udfs/scalar/DartGetTest.java | 2 +- .../dart/store/gcs/GcsDataStoreTest.java | 2 +- 79 files changed, 392 insertions(+), 372 deletions(-) rename dagger-common/src/main/java/io/odpf/dagger/common/metrics/{type/flink => manager}/CounterStatsManager.java (97%) rename dagger-common/src/main/java/io/odpf/dagger/common/metrics/{type/flink => manager}/GaugeStatsManager.java (98%) rename dagger-common/src/main/java/io/odpf/dagger/common/metrics/{type/flink => manager}/MeterStatsManager.java (98%) delete mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerStatsDClient.java delete mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/SerializedStatsDClientSupplier.java delete mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerCounterManager.java delete mode 100644 dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerMeterManager.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ChronologyOrderedSplitAssignerAspects.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfig.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterProvider.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java rename {dagger-common/src/main/java/io/odpf/dagger/common/metrics/type => dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters}/statsd/manager/DaggerGaugeManager.java (58%) rename {dagger-common/src/main/java/io/odpf/dagger/common/metrics/type => dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters}/statsd/manager/DaggerHistogramManager.java (51%) rename {dagger-common/src/main/java/io/odpf/dagger/common/metrics/type => dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager}/MeasurementManager.java (64%) rename {dagger-common/src/main/java/io/odpf/dagger/common/metrics/type => dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement}/Counter.java (56%) rename {dagger-common/src/main/java/io/odpf/dagger/common/metrics/type => dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement}/Gauge.java (82%) rename {dagger-common/src/main/java/io/odpf/dagger/common/metrics/type => dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement}/Histogram.java (78%) rename {dagger-common/src/main/java/io/odpf/dagger/common/metrics/type => dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement}/Meter.java (76%) create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/ComponentTags.java rename dagger-core/src/main/java/io/odpf/dagger/core/metrics/{telemetry/statsd => reporters/statsd/tags}/GlobalTags.java (57%) rename {dagger-common/src/main/java/io/odpf/dagger/common/metrics/type => dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters}/statsd/tags/StatsDTag.java (94%) delete mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/ComponentTags.java delete mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/CounterStatsManager.java similarity index 97% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManager.java rename to dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/CounterStatsManager.java index 2f4a86bba..3660ce000 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/CounterStatsManager.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.type.flink; +package io.odpf.dagger.common.metrics.manager; import io.odpf.dagger.common.metrics.aspects.AspectType; import io.odpf.dagger.common.metrics.aspects.Aspects; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/GaugeStatsManager.java similarity index 98% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManager.java rename to dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/GaugeStatsManager.java index 4e3eaf31e..a57f34f77 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/GaugeStatsManager.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.type.flink; +package io.odpf.dagger.common.metrics.manager; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.MetricGroup; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/MeterStatsManager.java similarity index 98% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManager.java rename to dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/MeterStatsManager.java index ffa5a8d37..7b995eca1 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/MeterStatsManager.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.type.flink; +package io.odpf.dagger.common.metrics.manager; import org.apache.flink.dropwizard.metrics.DropwizardHistogramWrapper; import org.apache.flink.dropwizard.metrics.DropwizardMeterWrapper; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerStatsDClient.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerStatsDClient.java deleted file mode 100644 index 758ec40bf..000000000 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/DaggerStatsDClient.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.odpf.dagger.common.metrics.type.statsd; - -import com.timgroup.statsd.NonBlockingStatsDClientBuilder; -import com.timgroup.statsd.StatsDClient; -import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; - -public class DaggerStatsDClient implements SerializedStatsDClientSupplier { - private final int port; - private final String hostname; - private final String[] constantTags; - private static StatsDClient statsDClient; - private static final String PREFIX = "statsd"; - private static final Logger LOGGER = LoggerFactory.getLogger(DaggerStatsDClient.class.getName()); - - public DaggerStatsDClient(String hostname, int port, StatsDTag[] globalTags) { - this.hostname = hostname; - this.port = port; - this.constantTags = Arrays.stream(globalTags) - .map(StatsDTag::getFormattedTag) - .toArray(String[]::new); - } - - @Override - public StatsDClient getClient() { - if (statsDClient == null) { - statsDClient = new NonBlockingStatsDClientBuilder() - .prefix(PREFIX) - .hostname(hostname) - .port(port) - .constantTags(constantTags) - .build(); - String message = String.format("Created new instance of StatsDClient for host=%s and port=%d", hostname, port); - LOGGER.info(message); - } - return statsDClient; - } -} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/SerializedStatsDClientSupplier.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/SerializedStatsDClientSupplier.java deleted file mode 100644 index a1f51d061..000000000 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/SerializedStatsDClientSupplier.java +++ /dev/null @@ -1,10 +0,0 @@ -package io.odpf.dagger.common.metrics.type.statsd; - -import com.timgroup.statsd.StatsDClient; - -import java.io.Serializable; - -@FunctionalInterface -public interface SerializedStatsDClientSupplier extends Serializable { - StatsDClient getClient(); -} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerCounterManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerCounterManager.java deleted file mode 100644 index 518c40d5c..000000000 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerCounterManager.java +++ /dev/null @@ -1,43 +0,0 @@ -package io.odpf.dagger.common.metrics.type.statsd.manager; - -import com.timgroup.statsd.StatsDClient; -import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; -import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; -import io.odpf.dagger.common.metrics.type.Counter; -import io.odpf.dagger.common.metrics.type.MeasurementManager; - -import java.util.ArrayList; - -public class DaggerCounterManager implements MeasurementManager, Counter { - private final StatsDClient statsDClient; - private String[] formattedTags; - - public DaggerCounterManager(SerializedStatsDClientSupplier statsDClientSupplier) { - this.statsDClient = statsDClientSupplier.getClient(); - } - - @Override - public void register(Aspects[] aspect, StatsDTag[] tags) { - register(tags); - } - - @Override - public void register(StatsDTag[] tags) { - ArrayList tagList = new ArrayList<>(); - for (StatsDTag measurementTag : tags) { - tagList.add(measurementTag.getFormattedTag()); - } - this.formattedTags = tagList.toArray(new String[0]); - } - - @Override - public void increment(Aspects aspect) { - statsDClient.increment(aspect.getValue(), formattedTags); - } - - @Override - public void decrement(Aspects aspect) { - statsDClient.decrement(aspect.getValue(), formattedTags); - } -} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerMeterManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerMeterManager.java deleted file mode 100644 index 8bb008868..000000000 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerMeterManager.java +++ /dev/null @@ -1,44 +0,0 @@ -package io.odpf.dagger.common.metrics.type.statsd.manager; - -import com.timgroup.statsd.StatsDClient; -import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; -import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; -import io.odpf.dagger.common.metrics.type.MeasurementManager; -import io.odpf.dagger.common.metrics.type.Meter; - -import java.util.ArrayList; -import java.util.function.Supplier; - -public class DaggerMeterManager implements MeasurementManager, Meter { - private final StatsDClient statsDClient; - private String[] formattedTags; - - public DaggerMeterManager(SerializedStatsDClientSupplier statsDClientSupplier) { - this.statsDClient = statsDClientSupplier.getClient(); - } - - @Override - public void register(Aspects[] aspect, StatsDTag[] tags) { - register(tags); - } - - @Override - public void register(StatsDTag[] tags) { - ArrayList tagList = new ArrayList<>(); - for (StatsDTag measurementTag : tags) { - tagList.add(measurementTag.getFormattedTag()); - } - this.formattedTags = tagList.toArray(new String[0]); - } - - @Override - public void markEvent(Aspects aspect) { - statsDClient.increment(aspect.getValue(), formattedTags); - } - - @Override - public void markEvent(Aspects aspect, long num) { - statsDClient.count(aspect.getValue(), num, formattedTags); - } -} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java index c1defe40a..1a0596efd 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.udfs; -import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; +import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; import org.apache.flink.table.functions.AggregateFunction; import org.apache.flink.table.functions.FunctionContext; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java index 4c04fc58e..f9ebde373 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.udfs; -import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; +import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.functions.ScalarFunction; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java index f0d0e3267..4c1eb054a 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.udfs; -import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; +import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.functions.TableFunction; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java index 0d45cf615..ddfea637c 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java @@ -1,5 +1,6 @@ package io.odpf.dagger.common.metrics.type.flink; +import io.odpf.dagger.common.metrics.manager.CounterStatsManager; import io.odpf.dagger.common.metrics.utils.TestAspects; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.MetricGroup; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java index 16231a7cd..3bcd4871c 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java @@ -1,5 +1,6 @@ package io.odpf.dagger.common.metrics.type.flink; +import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; import io.odpf.dagger.common.metrics.utils.TestAspects; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.MetricGroup; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java index 226809114..4a6c5d796 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java @@ -1,5 +1,6 @@ package io.odpf.dagger.common.metrics.type.flink; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.common.metrics.utils.TestAspects; import org.apache.flink.dropwizard.metrics.DropwizardMeterWrapper; import org.apache.flink.metrics.Histogram; diff --git a/dagger-core/build.gradle b/dagger-core/build.gradle index 6c95e910f..1ed74bc13 100644 --- a/dagger-core/build.gradle +++ b/dagger-core/build.gradle @@ -61,6 +61,7 @@ configurations { dependencies { minimalJar project(path: ':dagger-common', configuration: 'minimalCommonJar') minimalJar project(path: ':dagger-functions', configuration: 'minimalFunctionsJar') + minimalJar 'io.odpf:depot:0.1.5' compileOnly 'org.projectlombok:lombok:1.18.8' annotationProcessor 'org.projectlombok:lombok:1.18.8' @@ -71,7 +72,6 @@ dependencies { implementation 'org.apache.flink:flink-table:' + flinkVersion implementation 'org.apache.flink:flink-table-api-java-bridge_2.11:' + flinkVersion implementation 'org.apache.flink:flink-table-planner_2.11:' + flinkVersion - implementation 'io.odpf:depot:0.1.5' dependenciesJar project(path: ':dagger-common', configuration: 'dependenciesCommonJar') dependenciesJar project(path: ':dagger-functions', configuration: 'dependenciesFunctionsJar') diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java index 45d2a78e1..8bd482639 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core; -import io.odpf.dagger.common.metrics.type.statsd.DaggerStatsDClient; -import io.odpf.dagger.core.metrics.telemetry.statsd.DaggerStatsDClientProvider; +import io.odpf.dagger.core.metrics.reporters.statsd.DaggerStatsDReporterProvider; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.core.source.Stream; import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.datastream.DataStream; @@ -223,7 +223,7 @@ private void addSink(StreamInfo streamInfo) { List getStreams() { org.apache.flink.configuration.Configuration flinkConfiguration = (org.apache.flink.configuration.Configuration) this.executionEnvironment.getConfiguration(); - DaggerStatsDClient daggerStatsDClient = DaggerStatsDClientProvider.provide(flinkConfiguration, configuration); - return StreamsFactory.getStreams(configuration, stencilClientOrchestrator, daggerStatsDClient); + SerializedStatsDReporterSupplier statsDReporterSupplier = DaggerStatsDReporterProvider.provide(flinkConfiguration, configuration); + return StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDReporterSupplier); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ChronologyOrderedSplitAssignerAspects.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ChronologyOrderedSplitAssignerAspects.java new file mode 100644 index 000000000..555abccdd --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ChronologyOrderedSplitAssignerAspects.java @@ -0,0 +1,28 @@ +package io.odpf.dagger.core.metrics.aspects; + +import io.odpf.dagger.common.metrics.aspects.AspectType; +import io.odpf.dagger.common.metrics.aspects.Aspects; + +public enum ChronologyOrderedSplitAssignerAspects implements Aspects { + TOTAL_SPLITS_DISCOVERED("total_splits_discovered", AspectType.Gauge), + TOTAL_SPLITS_RECORDED("total_splits_recorded", AspectType.Gauge), + SPLITS_AWAITING_ASSIGNMENT("splits_awaiting_assignment", AspectType.Counter); + + ChronologyOrderedSplitAssignerAspects(String value, AspectType aspectType) { + this.value = value; + this.aspectType = aspectType; + } + + private final String value; + private final AspectType aspectType; + + @Override + public String getValue() { + return value; + } + + @Override + public AspectType getAspectType() { + return aspectType; + } +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfig.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfig.java new file mode 100644 index 000000000..aec1209fa --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfig.java @@ -0,0 +1,43 @@ +package io.odpf.dagger.core.metrics.reporters.statsd; + +import io.odpf.depot.config.MetricsConfig; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.Configuration; + +public class DaggerMetricsConfig implements MetricsConfig { + private static final String FLINK_STATSD_HOST_CONFIG_KEY = "metrics.reporter.stsd.host"; + private static final String DEFAULT_STATSD_HOST_VALUE = "localhost"; + private static final String FLINK_STATSD_PORT_CONFIG_KEY = "metrics.reporter.stsd.port"; + private static final int DEFAULT_STATSD_PORT_VALUE = 8125; + private final String hostName; + private final int port; + + public DaggerMetricsConfig(Configuration flinkConfiguration) { + ConfigOption hostConfigOption = ConfigOptions + .key(FLINK_STATSD_HOST_CONFIG_KEY) + .stringType() + .defaultValue(DEFAULT_STATSD_HOST_VALUE); + ConfigOption portConfigOption = ConfigOptions + .key(FLINK_STATSD_PORT_CONFIG_KEY) + .intType() + .defaultValue(DEFAULT_STATSD_PORT_VALUE); + this.hostName = flinkConfiguration.getString(hostConfigOption); + this.port = flinkConfiguration.getInteger(portConfigOption); + } + + @Override + public String getMetricStatsDHost() { + return hostName; + } + + @Override + public Integer getMetricStatsDPort() { + return port; + } + + @Override + public String getMetricStatsDTags() { + return null; + } +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterProvider.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterProvider.java new file mode 100644 index 000000000..60ee35c58 --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterProvider.java @@ -0,0 +1,41 @@ +package io.odpf.dagger.core.metrics.reporters.statsd; + +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.GlobalTags; +import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.metrics.StatsDReporterBuilder; +import org.apache.flink.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +import static io.odpf.dagger.core.utils.Constants.FLINK_JOB_ID_DEFAULT; +import static io.odpf.dagger.core.utils.Constants.FLINK_JOB_ID_KEY; + +public class DaggerStatsDReporterProvider { + private static StatsDReporter statsDReporter; + private static final Logger LOGGER = LoggerFactory.getLogger(DaggerStatsDReporterProvider.class.getName()); + + public static SerializedStatsDReporterSupplier provide(Configuration flinkConfiguration, io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { + if (statsDReporter == null) { + DaggerMetricsConfig daggerMetricsConfig = new DaggerMetricsConfig(flinkConfiguration); + String[] globalTags = generateGlobalTags(daggerConfiguration); + statsDReporter = StatsDReporterBuilder + .builder() + .withMetricConfig(daggerMetricsConfig) + .withExtraTags(globalTags) + .build(); + LOGGER.info("Instantiated new StatsDReporter"); + } + return () -> statsDReporter; + } + + private static String[] generateGlobalTags(io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { + StatsDTag[] globalTags = new StatsDTag[]{ + new StatsDTag(GlobalTags.JOB_ID, daggerConfiguration.getString(FLINK_JOB_ID_KEY, FLINK_JOB_ID_DEFAULT))}; + return Arrays.stream(globalTags) + .map(StatsDTag::getFormattedTag) + .toArray(String[]::new); + } +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java new file mode 100644 index 000000000..84ebbd23d --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java @@ -0,0 +1,10 @@ +package io.odpf.dagger.core.metrics.reporters.statsd; + +import io.odpf.depot.metrics.StatsDReporter; + +import java.io.Serializable; + +@FunctionalInterface +public interface SerializedStatsDReporterSupplier extends Serializable { + StatsDReporter getStatsDReporter(); +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java new file mode 100644 index 000000000..c85771c3d --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java @@ -0,0 +1,52 @@ +package io.odpf.dagger.core.metrics.reporters.statsd.manager; + +import io.odpf.dagger.common.metrics.aspects.Aspects; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.measurement.Counter; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; +import io.odpf.depot.metrics.StatsDReporter; + +import java.util.ArrayList; + +public class DaggerCounterManager implements MeasurementManager, Counter { + private final StatsDReporter statsDReporter; + private String[] formattedTags; + + public DaggerCounterManager(SerializedStatsDReporterSupplier statsDReporterSupplier) { + this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); + } + + @Override + public void register(Aspects[] aspect, StatsDTag[] tags) { + register(tags); + } + + @Override + public void register(StatsDTag[] tags) { + ArrayList tagList = new ArrayList<>(); + for (StatsDTag measurementTag : tags) { + tagList.add(measurementTag.getFormattedTag()); + } + this.formattedTags = tagList.toArray(new String[0]); + } + + @Override + public void increment(Aspects aspect) { + increment(aspect, 1L); + } + + @Override + public void increment(Aspects aspect, long positiveCount) { + statsDReporter.captureCount(aspect.getValue(), positiveCount, formattedTags); + } + + @Override + public void decrement(Aspects aspect) { + decrement(aspect, -1L); + } + + @Override + public void decrement(Aspects aspect, long negativeCount) { + statsDReporter.captureCount(aspect.getValue(), negativeCount, formattedTags); + } +} diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerGaugeManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java similarity index 58% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerGaugeManager.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java index 2e76d5c93..bcadd7f85 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerGaugeManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java @@ -1,20 +1,20 @@ -package io.odpf.dagger.common.metrics.type.statsd.manager; +package io.odpf.dagger.core.metrics.reporters.statsd.manager; -import com.timgroup.statsd.StatsDClient; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; -import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; -import io.odpf.dagger.common.metrics.type.Gauge; -import io.odpf.dagger.common.metrics.type.MeasurementManager; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.measurement.Gauge; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; +import io.odpf.depot.metrics.StatsDReporter; import java.util.ArrayList; + public class DaggerGaugeManager implements MeasurementManager, Gauge { - private final StatsDClient statsDClient; + private final StatsDReporter statsDReporter; private String[] formattedTags; - public DaggerGaugeManager(SerializedStatsDClientSupplier statsDClientSupplier) { - this.statsDClient = statsDClientSupplier.getClient(); + public DaggerGaugeManager(SerializedStatsDReporterSupplier statsDReporterSupplier) { + this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); } @Override @@ -33,7 +33,7 @@ public void register(StatsDTag[] tags) { @Override public void registerLong(Aspects aspect, long gaugeValue) { - statsDClient.gauge(aspect.getValue(), gaugeValue, formattedTags); + statsDReporter.getClient().gauge(aspect.getValue(), gaugeValue, formattedTags); } @Override @@ -43,6 +43,6 @@ public void registerString(Aspects aspect, String gaugeValue) { @Override public void registerDouble(Aspects aspect, double gaugeValue) { - statsDClient.gauge(aspect.getValue(), gaugeValue, formattedTags); + statsDReporter.getClient().gauge(aspect.getValue(), gaugeValue, formattedTags); } } diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java similarity index 51% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java index 2f7c90327..b5f6e233a 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/manager/DaggerHistogramManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java @@ -1,20 +1,19 @@ -package io.odpf.dagger.common.metrics.type.statsd.manager; +package io.odpf.dagger.core.metrics.reporters.statsd.manager; -import com.timgroup.statsd.StatsDClient; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; -import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; -import io.odpf.dagger.common.metrics.type.Histogram; -import io.odpf.dagger.common.metrics.type.MeasurementManager; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.measurement.Histogram; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; +import io.odpf.depot.metrics.StatsDReporter; import java.util.ArrayList; public class DaggerHistogramManager implements MeasurementManager, Histogram { - private final StatsDClient statsDClient; + private final StatsDReporter statsDReporter; private String[] formattedTags; - public DaggerHistogramManager(SerializedStatsDClientSupplier statsDClientSupplier) { - this.statsDClient = statsDClientSupplier.getClient(); + public DaggerHistogramManager(SerializedStatsDReporterSupplier statsDReporterSupplier) { + this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); } @Override @@ -33,11 +32,11 @@ public void register(StatsDTag[] tags) { @Override public void recordValue(Aspects aspect, long value) { - statsDClient.histogram(aspect.getValue(), value, formattedTags); + statsDReporter.getClient().histogram(aspect.getValue(), value, formattedTags); } @Override public void recordValue(Aspects aspect, double value) { - statsDClient.histogram(aspect.getValue(), value, formattedTags); + statsDReporter.getClient().histogram(aspect.getValue(), value, formattedTags); } } diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java similarity index 64% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java index 7c7044563..ee46fa3d6 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/MeasurementManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java @@ -1,7 +1,7 @@ -package io.odpf.dagger.common.metrics.type; +package io.odpf.dagger.core.metrics.reporters.statsd.manager; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; import java.io.Serializable; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Counter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Counter.java similarity index 56% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Counter.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Counter.java index db4b43a2a..0b0668aac 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Counter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Counter.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.type; +package io.odpf.dagger.core.metrics.reporters.statsd.measurement; import io.odpf.dagger.common.metrics.aspects.Aspects; @@ -7,5 +7,9 @@ public interface Counter extends Serializable { void increment(Aspects aspect); + void increment(Aspects aspect, long num); + void decrement(Aspects aspect); + + void decrement(Aspects aspect, long num); } diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java similarity index 82% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java index 72b7cd9ef..8c50edaf4 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Gauge.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.type; +package io.odpf.dagger.core.metrics.reporters.statsd.measurement; import io.odpf.dagger.common.metrics.aspects.Aspects; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Histogram.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Histogram.java similarity index 78% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Histogram.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Histogram.java index 5a4e3894e..338f20d1d 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Histogram.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Histogram.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.type; +package io.odpf.dagger.core.metrics.reporters.statsd.measurement; import io.odpf.dagger.common.metrics.aspects.Aspects; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Meter.java similarity index 76% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Meter.java index cdc7e62ed..14bc4830b 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/Meter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Meter.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.type; +package io.odpf.dagger.core.metrics.reporters.statsd.measurement; import io.odpf.dagger.common.metrics.aspects.Aspects; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/ComponentTags.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/ComponentTags.java new file mode 100644 index 000000000..39d156f5d --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/ComponentTags.java @@ -0,0 +1,28 @@ +package io.odpf.dagger.core.metrics.reporters.statsd.tags; + +public class ComponentTags { + private static StatsDTag[] parquetReaderTags; + private static StatsDTag[] splitAssignerTags; + private static final String COMPONENT_TAG_KEY = "component"; + private static final String DAGGER_SOURCE_TAG_KEY = "dagger_source"; + + public static StatsDTag[] getParquetReaderTags() { + if (parquetReaderTags == null) { + parquetReaderTags = new StatsDTag[]{ + new StatsDTag(COMPONENT_TAG_KEY, "parquet_reader"), + new StatsDTag(DAGGER_SOURCE_TAG_KEY, "parquet_source") + }; + } + return parquetReaderTags; + } + + public static StatsDTag[] getSplitAssignerTags() { + if (splitAssignerTags == null) { + splitAssignerTags = new StatsDTag[]{ + new StatsDTag(COMPONENT_TAG_KEY, "split_assigner"), + new StatsDTag(DAGGER_SOURCE_TAG_KEY, "parquet_source") + }; + } + return splitAssignerTags; + } +} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/GlobalTags.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/GlobalTags.java similarity index 57% rename from dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/GlobalTags.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/GlobalTags.java index bdb5703d8..2b3d9b143 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/GlobalTags.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/GlobalTags.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.core.metrics.telemetry.statsd; +package io.odpf.dagger.core.metrics.reporters.statsd.tags; public class GlobalTags { public static final String JOB_ID = "job_id"; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/tags/StatsDTag.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java similarity index 94% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/tags/StatsDTag.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java index f4ea0c0e1..1b2cbefb5 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/type/statsd/tags/StatsDTag.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.type.statsd.tags; +package io.odpf.dagger.core.metrics.reporters.statsd.tags; import org.apache.flink.util.Preconditions; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/ComponentTags.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/ComponentTags.java deleted file mode 100644 index 16d357535..000000000 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/ComponentTags.java +++ /dev/null @@ -1,12 +0,0 @@ -package io.odpf.dagger.core.metrics.telemetry.statsd; - -import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; - -public class ComponentTags { - public static StatsDTag[] getParquetReaderTags() { - return new StatsDTag[]{ - new StatsDTag("component", "parquet_reader"), - new StatsDTag("dagger_source", "parquet_source") - }; - } -} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java deleted file mode 100644 index fc9fc8ccd..000000000 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/telemetry/statsd/DaggerStatsDClientProvider.java +++ /dev/null @@ -1,45 +0,0 @@ -package io.odpf.dagger.core.metrics.telemetry.statsd; - -import io.odpf.dagger.common.metrics.type.statsd.DaggerStatsDClient; -import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; -import org.apache.flink.configuration.ConfigOption; -import org.apache.flink.configuration.ConfigOptions; -import org.apache.flink.configuration.Configuration; - -import static io.odpf.dagger.core.utils.Constants.FLINK_JOB_ID_DEFAULT; -import static io.odpf.dagger.core.utils.Constants.FLINK_JOB_ID_KEY; - -public class DaggerStatsDClientProvider { - - private static final String FLINK_STATSD_HOST_CONFIG_KEY = "metrics.reporter.stsd.host"; - private static final String DEFAULT_STATSD_HOST_VALUE = "localhost"; - private static final String FLINK_STATSD_PORT_CONFIG_KEY = "metrics.reporter.stsd.port"; - private static final int DEFAULT_STATSD_PORT_VALUE = 8125; - private static StatsDTag[] globalTags; - - public static DaggerStatsDClient provide(Configuration flinkConfiguration, io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { - ConfigOption hostConfigOption = ConfigOptions - .key(FLINK_STATSD_HOST_CONFIG_KEY) - .stringType() - .defaultValue(DEFAULT_STATSD_HOST_VALUE); - ConfigOption portConfigOption = ConfigOptions - .key(FLINK_STATSD_PORT_CONFIG_KEY) - .intType() - .defaultValue(DEFAULT_STATSD_PORT_VALUE); - - String hostname = flinkConfiguration.getString(hostConfigOption); - int port = flinkConfiguration.getInteger(portConfigOption); - StatsDTag[] globalTags = generateGlobalTags(daggerConfiguration); - - return new DaggerStatsDClient(hostname, port, globalTags); - } - - public static StatsDTag[] generateGlobalTags(io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { - if (globalTags == null) { - globalTags = new StatsDTag[]{ - new StatsDTag(GlobalTags.JOB_ID, daggerConfiguration.getString(FLINK_JOB_ID_KEY, FLINK_JOB_ID_DEFAULT)), - }; - } - return globalTags; - } -} diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java index b24da4d37..274122962 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java @@ -7,7 +7,7 @@ import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; import io.odpf.dagger.core.exception.InvalidConfigurationException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.common.serde.typehandler.TypeHandler; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java index 833367d13..76a0d6246 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java @@ -1,6 +1,6 @@ package io.odpf.dagger.core.processors.common; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import java.time.Instant; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java index 43325e921..cc6d2bbb2 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java @@ -8,7 +8,7 @@ import com.google.protobuf.Descriptors; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java index db6c36b65..f29b39a3d 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java @@ -1,6 +1,6 @@ package io.odpf.dagger.core.processors.external.es; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.common.PostResponseTelemetry; import io.odpf.dagger.core.processors.common.RowManager; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java index d02fde809..d0f81c416 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java @@ -9,7 +9,7 @@ import com.jayway.jsonpath.PathNotFoundException; import io.odpf.dagger.common.serde.typehandler.TypeHandler; import io.odpf.dagger.common.serde.typehandler.TypeHandlerFactory; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.exception.HttpFailureException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java index 7e79d2c71..4a2a45398 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java @@ -13,7 +13,7 @@ import io.odpf.dagger.core.exception.ChannelNotAvailableException; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; import io.odpf.dagger.core.exception.InvalidGrpcBodyException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.external.grpc.client.GrpcClient; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java index f204fc8ca..4e1fa7e9d 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.grpc; import io.odpf.dagger.core.exception.GrpcFailureException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.ColumnNameManager; import io.odpf.dagger.core.processors.common.OutputMapping; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java index 8b9fd57aa..71c404b56 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.http; import io.odpf.dagger.core.exception.InvalidHttpVerbException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.common.DescriptorManager; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java index 388406504..89441b746 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.http; import io.odpf.dagger.core.exception.HttpFailureException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.ColumnNameManager; import io.odpf.dagger.core.processors.common.OutputMapping; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java index e81166916..7c20ec595 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.pg; import io.odpf.dagger.core.exception.InvalidConfigurationException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.common.PostResponseTelemetry; import io.odpf.dagger.core.processors.common.RowManager; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java index 4e7b90984..d94fc29b1 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.pg; import io.odpf.dagger.core.exception.HttpFailureException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.ColumnNameManager; import io.odpf.dagger.core.processors.common.PostResponseTelemetry; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java index 452bc6a00..c10e0e52d 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java @@ -5,7 +5,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.LongbowReaderAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.reporters.ErrorReporterFactory; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java index 1a01c64a0..0754df6fd 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java @@ -5,7 +5,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.LongbowWriterAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.reporters.ErrorReporterFactory; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java index f26e18289..adfac16e9 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java @@ -6,7 +6,7 @@ import io.odpf.dagger.core.metrics.telemetry.TelemetryPublisher; import io.odpf.dagger.core.metrics.telemetry.TelemetrySubscriber; -import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; +import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; import io.odpf.dagger.core.metrics.aspects.TelemetryAspects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java index 63e40c319..231bf9b93 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.source; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.core.exception.InvalidDaggerSourceException; import io.odpf.dagger.core.source.config.StreamConfig; @@ -17,8 +17,8 @@ public class DaggerSourceFactory { - public static DaggerSource create(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDClientSupplier statsDClientSupplier) { - List> daggerSources = getDaggerSources(streamConfig, configuration, deserializer, statsDClientSupplier); + public static DaggerSource create(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDReporterSupplier statsDReporterSupplier) { + List> daggerSources = getDaggerSources(streamConfig, configuration, deserializer, statsDReporterSupplier); return daggerSources.stream() .filter(DaggerSource::canBuild) .findFirst() @@ -29,10 +29,10 @@ public static DaggerSource create(StreamConfig streamConfig, Configuration }); } - private static List> getDaggerSources(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDClientSupplier statsDClientSupplier) { + private static List> getDaggerSources(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDReporterSupplier statsDReporterSupplier) { KafkaDaggerSource kafkaDaggerSource = new KafkaDaggerSource(streamConfig, configuration, deserializer); FlinkKafkaConsumerDaggerSource flinkKafkaConsumerDaggerSource = new FlinkKafkaConsumerDaggerSource(streamConfig, configuration, deserializer); - ParquetDaggerSource parquetDaggerSource = new ParquetDaggerSource(streamConfig, configuration, deserializer, statsDClientSupplier); + ParquetDaggerSource parquetDaggerSource = new ParquetDaggerSource(streamConfig, configuration, deserializer, statsDReporterSupplier); return Stream.of(kafkaDaggerSource, flinkKafkaConsumerDaggerSource, parquetDaggerSource) .collect(Collectors.toList()); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java index cbf3e3e7c..638b7f2b4 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java @@ -2,7 +2,7 @@ import io.odpf.dagger.common.configuration.Configuration; import io.odpf.dagger.common.core.StencilClientOrchestrator; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.core.deserializer.DaggerDeserializerFactory; import io.odpf.dagger.core.source.config.StreamConfig; @@ -33,18 +33,18 @@ public static class Builder { private final StreamConfig streamConfig; private final Configuration configuration; private final StencilClientOrchestrator stencilClientOrchestrator; - private final SerializedStatsDClientSupplier statsDClientSupplier; + private final SerializedStatsDReporterSupplier statsDReporterSupplier; - public Builder(StreamConfig streamConfig, Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator, SerializedStatsDClientSupplier statsDClientSupplier) { + public Builder(StreamConfig streamConfig, Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator, SerializedStatsDReporterSupplier statsDReporterSupplier) { this.streamConfig = streamConfig; this.configuration = configuration; this.stencilClientOrchestrator = stencilClientOrchestrator; - this.statsDClientSupplier = statsDClientSupplier; + this.statsDReporterSupplier = statsDReporterSupplier; } public Stream build() { DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator); - DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, daggerDeserializer, statsDClientSupplier); + DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, daggerDeserializer, statsDReporterSupplier); return new Stream(daggerSource, streamConfig.getSchemaTable()); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/StreamsFactory.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/StreamsFactory.java index de3886e57..034e92c09 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/StreamsFactory.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/StreamsFactory.java @@ -2,19 +2,19 @@ import io.odpf.dagger.common.configuration.Configuration; import io.odpf.dagger.common.core.StencilClientOrchestrator; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.core.source.config.StreamConfig; import java.util.ArrayList; import java.util.List; public class StreamsFactory { - public static List getStreams(Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator, SerializedStatsDClientSupplier statsDClientSupplier) { + public static List getStreams(Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator, SerializedStatsDReporterSupplier statsDReporterSupplier) { StreamConfig[] streamConfigs = StreamConfig.parse(configuration); ArrayList streams = new ArrayList<>(); for (StreamConfig streamConfig : streamConfigs) { - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplier); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplier); streams.add(builder.build()); } return streams; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java index 499ae63f4..2f55876da 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.source.parquet; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.DaggerConfigurationException; @@ -34,15 +34,15 @@ public class ParquetDaggerSource implements DaggerSource { private final DaggerDeserializer deserializer; private final StreamConfig streamConfig; private final Configuration configuration; - private final SerializedStatsDClientSupplier statsDClientSupplier; + private final SerializedStatsDReporterSupplier statsDReporterSupplier; private static final SourceType SUPPORTED_SOURCE_TYPE = BOUNDED; private static final SourceName SUPPORTED_SOURCE_NAME = PARQUET_SOURCE; - public ParquetDaggerSource(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDClientSupplier statsDClientSupplier) { + public ParquetDaggerSource(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDReporterSupplier statsDReporterSupplier) { this.streamConfig = streamConfig; this.configuration = configuration; this.deserializer = deserializer; - this.statsDClientSupplier = statsDClientSupplier; + this.statsDReporterSupplier = statsDReporterSupplier; } @Override @@ -92,6 +92,7 @@ private FileSplitAssigner.Provider buildParquetFileSplitAssignerProvider() { ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder chronologyOrderedSplitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder() .addTimeRanges(streamConfig.getParquetFileDateRange()) + .addStatsDReporterSupplier(statsDReporterSupplier) .addPathParser(new HourDatePathParser()); return chronologyOrderedSplitAssignerBuilder::build; case EARLIEST_INDEX_FIRST: @@ -102,13 +103,12 @@ private FileSplitAssigner.Provider buildParquetFileSplitAssignerProvider() { private ParquetFileRecordFormat buildParquetFileRecordFormat() { SimpleGroupDeserializer simpleGroupDeserializer = (SimpleGroupDeserializer) deserializer; - ReaderProvider parquetFileReaderProvider = new ParquetReader.ParquetReaderProvider(simpleGroupDeserializer, statsDClientSupplier); + ReaderProvider parquetFileReaderProvider = new ParquetReader.ParquetReaderProvider(simpleGroupDeserializer, statsDReporterSupplier); ParquetFileRecordFormat.Builder parquetFileRecordFormatBuilder = ParquetFileRecordFormat.Builder.getInstance(); Supplier> typeInformationProvider = (Supplier> & Serializable) simpleGroupDeserializer::getProducedType; return parquetFileRecordFormatBuilder .setParquetFileReaderProvider(parquetFileReaderProvider) .setTypeInformationProvider(typeInformationProvider) - .setStatsDClientSupplier(statsDClientSupplier) .build(); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java index a9775c545..9828dfabb 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java @@ -2,7 +2,6 @@ import static com.google.api.client.util.Preconditions.checkArgument; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.core.source.parquet.reader.ReaderProvider; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; @@ -15,13 +14,10 @@ public class ParquetFileRecordFormat implements FileRecordFormat { private final ReaderProvider parquetFileReaderProvider; private final Supplier> typeInformationProvider; - private final SerializedStatsDClientSupplier statsDClientSupplier; - private ParquetFileRecordFormat(ReaderProvider parquetFileReaderProvider, Supplier> - typeInformationProvider, SerializedStatsDClientSupplier statsDClientSupplier) { + private ParquetFileRecordFormat(ReaderProvider parquetFileReaderProvider, Supplier> typeInformationProvider) { this.parquetFileReaderProvider = parquetFileReaderProvider; this.typeInformationProvider = typeInformationProvider; - this.statsDClientSupplier = statsDClientSupplier; } @Override @@ -48,7 +44,6 @@ public TypeInformation getProducedType() { public static class Builder { private ReaderProvider parquetFileReaderProvider; private Supplier> typeInformationProvider; - private SerializedStatsDClientSupplier statsDClientSupplier; public static Builder getInstance() { return new Builder(); @@ -69,16 +64,10 @@ public Builder setTypeInformationProvider(Supplier> typeInf return this; } - public Builder setStatsDClientSupplier(SerializedStatsDClientSupplier statsDClientSupplier) { - this.statsDClientSupplier = statsDClientSupplier; - return this; - } - public ParquetFileRecordFormat build() { checkArgument(parquetFileReaderProvider != null, "ReaderProvider is required but is set as null"); checkArgument(typeInformationProvider != null, "TypeInformationProvider is required but is set as null"); - checkArgument(statsDClientSupplier != null, "StatsDClientSupplier object is required but is set as null"); - return new ParquetFileRecordFormat(parquetFileReaderProvider, typeInformationProvider, statsDClientSupplier); + return new ParquetFileRecordFormat(parquetFileReaderProvider, typeInformationProvider); } } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java index 411548778..139b40f9e 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java @@ -1,9 +1,9 @@ package io.odpf.dagger.core.source.parquet.reader; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; -import io.odpf.dagger.common.metrics.type.statsd.manager.DaggerCounterManager; -import io.odpf.dagger.common.metrics.type.statsd.manager.DaggerHistogramManager; -import io.odpf.dagger.common.metrics.type.statsd.tags.StatsDTag; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.manager.DaggerCounterManager; +import io.odpf.dagger.core.metrics.reporters.statsd.manager.DaggerHistogramManager; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.ParquetFileSourceReaderInitializationException; import io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects; @@ -29,7 +29,7 @@ import java.io.IOException; import java.time.Instant; -import static io.odpf.dagger.core.metrics.telemetry.statsd.ComponentTags.getParquetReaderTags; +import static io.odpf.dagger.core.metrics.reporters.statsd.tags.ComponentTags.getParquetReaderTags; public class ParquetReader implements FileRecordFormat.Reader { private final Path hadoopFilePath; @@ -46,22 +46,22 @@ public class ParquetReader implements FileRecordFormat.Reader { private static final Logger LOGGER = LoggerFactory.getLogger(ParquetReader.class.getName()); private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDeserializer, ParquetFileReader - parquetFileReader, SerializedStatsDClientSupplier statsDClientSupplier) throws IOException { + parquetFileReader, SerializedStatsDReporterSupplier statsDReporterSupplier) throws IOException { this.hadoopFilePath = hadoopFilePath; this.simpleGroupDeserializer = simpleGroupDeserializer; this.parquetFileReader = parquetFileReader; this.schema = this.parquetFileReader.getFileMetaData().getSchema(); this.isRecordReaderInitialized = false; this.totalEmittedRowCount = 0L; - this.registerTagsWithMeasurementManagers(statsDClientSupplier); + this.registerTagsWithMeasurementManagers(statsDReporterSupplier); daggerCounterManager.increment(ParquetReaderAspects.READER_CREATED); } - private void registerTagsWithMeasurementManagers(SerializedStatsDClientSupplier statsDClientSupplier) { + private void registerTagsWithMeasurementManagers(SerializedStatsDReporterSupplier statsDReporterSupplier) { StatsDTag[] parquetReaderTags = getParquetReaderTags(); - this.daggerCounterManager = new DaggerCounterManager(statsDClientSupplier); + this.daggerCounterManager = new DaggerCounterManager(statsDReporterSupplier); this.daggerCounterManager.register(parquetReaderTags); - this.daggerHistogramManager = new DaggerHistogramManager(statsDClientSupplier); + this.daggerHistogramManager = new DaggerHistogramManager(statsDReporterSupplier); this.daggerHistogramManager.register(parquetReaderTags); } @@ -147,11 +147,11 @@ public CheckpointedPosition getCheckpointedPosition() { public static class ParquetReaderProvider implements ReaderProvider { private final SimpleGroupDeserializer simpleGroupDeserializer; - private final SerializedStatsDClientSupplier statsDClientSupplier; + private final SerializedStatsDReporterSupplier statsDReporterSupplier; - public ParquetReaderProvider(SimpleGroupDeserializer simpleGroupDeserializer, SerializedStatsDClientSupplier statsDClientSupplier) { + public ParquetReaderProvider(SimpleGroupDeserializer simpleGroupDeserializer, SerializedStatsDReporterSupplier statsDReporterSupplier) { this.simpleGroupDeserializer = simpleGroupDeserializer; - this.statsDClientSupplier = statsDClientSupplier; + this.statsDReporterSupplier = statsDReporterSupplier; } @Override @@ -160,7 +160,7 @@ public ParquetReader getReader(String filePath) { Configuration conf = new Configuration(); Path hadoopFilePath = new Path(filePath); ParquetFileReader parquetFileReader = ParquetFileReader.open(HadoopInputFile.fromPath(hadoopFilePath, conf)); - return new ParquetReader(hadoopFilePath, simpleGroupDeserializer, parquetFileReader, statsDClientSupplier); + return new ParquetReader(hadoopFilePath, simpleGroupDeserializer, parquetFileReader, statsDReporterSupplier); } catch (IOException | RuntimeException ex) { throw new ParquetFileSourceReaderInitializationException(ex); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java index 9c0fef179..9d9f00674 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java @@ -1,5 +1,8 @@ package io.odpf.dagger.core.source.parquet.splitassigner; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.manager.DaggerGaugeManager; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; import io.odpf.dagger.core.exception.PathParserNotProvidedException; import io.odpf.dagger.core.source.config.models.TimeRangePool; import io.odpf.dagger.core.source.parquet.path.PathParser; @@ -16,19 +19,36 @@ import java.util.concurrent.PriorityBlockingQueue; import java.util.stream.Collectors; +import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.SPLITS_AWAITING_ASSIGNMENT; +import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.TOTAL_SPLITS_DISCOVERED; +import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.TOTAL_SPLITS_RECORDED; +import static io.odpf.dagger.core.metrics.reporters.statsd.tags.ComponentTags.getSplitAssignerTags; + public class ChronologyOrderedSplitAssigner implements FileSplitAssigner { private final PriorityBlockingQueue unassignedSplits; private static final int INITIAL_DEFAULT_CAPACITY = 11; private PathParser pathParser; private TimeRangePool timeRangePool; + private DaggerGaugeManager daggerGaugeManager; - private ChronologyOrderedSplitAssigner(Collection fileSourceSplits, PathParser pathParser, TimeRangePool timeRangePool) { + private ChronologyOrderedSplitAssigner(Collection fileSourceSplits, PathParser pathParser, + TimeRangePool timeRangePool, SerializedStatsDReporterSupplier statsDReporterSupplier) { + this.registerTagsWithMeasurementManagers(statsDReporterSupplier); + daggerGaugeManager.registerLong(TOTAL_SPLITS_DISCOVERED, fileSourceSplits.size()); this.pathParser = pathParser; this.timeRangePool = timeRangePool; this.unassignedSplits = new PriorityBlockingQueue<>(INITIAL_DEFAULT_CAPACITY, getFileSourceSplitComparator()); for (FileSourceSplit split : fileSourceSplits) { validateAndAddSplits(split); } + daggerGaugeManager.registerLong(TOTAL_SPLITS_RECORDED, unassignedSplits.size()); + daggerGaugeManager.registerLong(SPLITS_AWAITING_ASSIGNMENT, unassignedSplits.size()); + } + + private void registerTagsWithMeasurementManagers(SerializedStatsDReporterSupplier statsDReporterSupplier) { + StatsDTag[] splitAssignerTags = getSplitAssignerTags(); + daggerGaugeManager = new DaggerGaugeManager(statsDReporterSupplier); + daggerGaugeManager.register(splitAssignerTags); } @Override @@ -37,6 +57,7 @@ public Optional getNext(@Nullable String hostname) { if (instantEnrichedSplit == null) { return Optional.empty(); } + daggerGaugeManager.registerLong(SPLITS_AWAITING_ASSIGNMENT, unassignedSplits.size()); return Optional.of(instantEnrichedSplit.getFileSourceSplit()); } @@ -83,6 +104,7 @@ private Comparator getFileSourceSplitComparator() { public static class ChronologyOrderedSplitAssignerBuilder implements Serializable { private PathParser pathParser; private TimeRangePool parquetFileDateRange; + private SerializedStatsDReporterSupplier supplier; public ChronologyOrderedSplitAssignerBuilder addPathParser(PathParser parser) { this.pathParser = parser; @@ -94,11 +116,16 @@ public ChronologyOrderedSplitAssignerBuilder addTimeRanges(TimeRangePool timeRan return this; } + public ChronologyOrderedSplitAssignerBuilder addStatsDReporterSupplier(SerializedStatsDReporterSupplier statsDReporterSupplier) { + this.supplier = statsDReporterSupplier; + return this; + } + public ChronologyOrderedSplitAssigner build(Collection fileSourceSplits) { if (pathParser == null) { throw new PathParserNotProvidedException("Path parser is null"); } - return new ChronologyOrderedSplitAssigner(fileSourceSplits, pathParser, parquetFileDateRange); + return new ChronologyOrderedSplitAssigner(fileSourceSplits, pathParser, parquetFileDateRange, supplier); } } } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java index c40d28e56..f338c5e14 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java @@ -4,7 +4,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.consumer.TestEnumType; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java index 21c1dd594..2e6494fed 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java @@ -7,7 +7,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java index 8340c6d0a..3fa62417e 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java @@ -2,7 +2,7 @@ import com.google.protobuf.Descriptors; import com.jayway.jsonpath.PathNotFoundException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestEnrichedBookingLogMessage; import io.odpf.dagger.consumer.TestProfile; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java index 8cb9d0124..eaf1205ad 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java @@ -4,7 +4,7 @@ import com.google.protobuf.DynamicMessage; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestGrpcRequest; import io.odpf.dagger.core.exception.InvalidConfigurationException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java index 4d7d7974c..6a4aa43c6 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java @@ -5,7 +5,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.jayway.jsonpath.PathNotFoundException; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestGrpcResponse; import io.odpf.dagger.consumer.TestLocation; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java index d2f8c32ab..b10a51da8 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java @@ -3,7 +3,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.exception.InvalidHttpVerbException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java index 4e1cec53b..68b44ef04 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java @@ -2,7 +2,7 @@ import com.google.protobuf.Descriptors; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestSurgeFactorLogMessage; import io.odpf.dagger.core.exception.HttpFailureException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java index 2aa5a2e78..86b44f4c5 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java @@ -3,7 +3,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java index 6de7ab52e..fe11b799e 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java @@ -2,7 +2,7 @@ import com.google.protobuf.Descriptors; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestSurgeFactorLogMessage; import io.odpf.dagger.core.exception.HttpFailureException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java index 345209ef3..650a10945 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java @@ -4,7 +4,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.telemetry.TelemetrySubscriber; import io.odpf.dagger.core.processors.longbow.LongbowSchema; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java index 5e9561cbc..f36eab349 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java @@ -5,7 +5,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.LongbowWriterAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.telemetry.TelemetrySubscriber; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java index c71b827ad..cf623430a 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java @@ -6,7 +6,7 @@ import org.apache.flink.metrics.groups.OperatorMetricGroup; import org.apache.flink.types.Row; -import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; +import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; import io.odpf.dagger.core.metrics.telemetry.TelemetryPublisher; import org.junit.Before; import org.junit.Test; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java index 8bf0ebc73..6ba8432c7 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java @@ -1,8 +1,7 @@ package io.odpf.dagger.core.source; -import com.timgroup.statsd.StatsDClient; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.common.serde.json.deserialization.JsonDeserializer; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.common.serde.proto.deserialization.ProtoDeserializer; @@ -14,6 +13,7 @@ import io.odpf.dagger.core.source.flinkkafkaconsumer.FlinkKafkaConsumerDaggerSource; import io.odpf.dagger.core.source.kafka.KafkaDaggerSource; import io.odpf.dagger.core.source.parquet.ParquetDaggerSource; +import io.odpf.depot.metrics.StatsDReporter; import org.apache.flink.types.Row; import org.junit.Before; import org.junit.Test; @@ -32,7 +32,7 @@ public class DaggerSourceFactoryTest { @Mock private Configuration configuration; - private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); @Before public void setUp() throws Exception { @@ -43,7 +43,7 @@ public void setUp() throws Exception { public void shouldReturnKafkaDaggerSourceWhenConfigured() { ProtoDeserializer deserializer = Mockito.mock(ProtoDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.KAFKA_SOURCE, SourceType.UNBOUNDED)}); - DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDClientSupplierMock); + DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDReporterSupplierMock); assertTrue(daggerSource instanceof KafkaDaggerSource); } @@ -52,7 +52,7 @@ public void shouldReturnKafkaDaggerSourceWhenConfigured() { public void shouldReturnFlinkKafkaConsumerDaggerSourceWhenConfigured() { JsonDeserializer deserializer = Mockito.mock(JsonDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.KAFKA_CONSUMER, SourceType.UNBOUNDED)}); - DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDClientSupplierMock); + DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDReporterSupplierMock); assertTrue(daggerSource instanceof FlinkKafkaConsumerDaggerSource); } @@ -61,7 +61,7 @@ public void shouldReturnFlinkKafkaConsumerDaggerSourceWhenConfigured() { public void shouldReturnParquetDaggerSourceWhenConfigured() { SimpleGroupDeserializer deserializer = Mockito.mock(SimpleGroupDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED)}); - DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDClientSupplierMock); + DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDReporterSupplierMock); assertTrue(daggerSource instanceof ParquetDaggerSource); } @@ -71,6 +71,6 @@ public void shouldThrowRuntimeExceptionIfNoDaggerSourceCouldBeCreatedAsPerConfig SimpleGroupDeserializer deserializer = Mockito.mock(SimpleGroupDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.UNBOUNDED)}); - assertThrows(InvalidDaggerSourceException.class, () -> DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDClientSupplierMock)); + assertThrows(InvalidDaggerSourceException.class, () -> DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDReporterSupplierMock)); } } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamTest.java index 78d120195..a7366cd11 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamTest.java @@ -1,9 +1,8 @@ package io.odpf.dagger.core.source; -import com.timgroup.statsd.StatsDClient; import io.odpf.dagger.common.configuration.Configuration; import io.odpf.dagger.common.core.StencilClientOrchestrator; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.source.config.StreamConfig; import io.odpf.dagger.core.source.config.models.SourceDetails; @@ -12,6 +11,7 @@ import io.odpf.dagger.core.source.flinkkafkaconsumer.FlinkKafkaConsumerDaggerSource; import io.odpf.dagger.core.source.kafka.KafkaDaggerSource; import io.odpf.dagger.core.source.parquet.ParquetDaggerSource; +import io.odpf.depot.metrics.StatsDReporter; import io.odpf.stencil.client.StencilClient; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -56,7 +56,7 @@ public class StreamTest { @Mock private DaggerSource mockDaggerSource; - private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); @Before public void setup() { @@ -73,7 +73,7 @@ public void shouldBeAbleToBuildAStreamWithKafkaDaggerSourceAndProtoSchema() { when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof KafkaDaggerSource); @@ -98,7 +98,7 @@ public void shouldBeAbleToBuildAStreamWithFlinkKafkaConsumerDaggerSourceAndProto when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof FlinkKafkaConsumerDaggerSource); @@ -111,7 +111,7 @@ public void shouldBeAbleToBuildAStreamWithKafkaDaggerSourceAndJsonSchema() { when(streamConfig.getSchemaTable()).thenReturn("data_stream"); when(streamConfig.getJsonSchema()).thenReturn("{ \"$schema\": \"https://json-schema.org/draft/2020-12/schema\", \"$id\": \"https://example.com/product.schema.json\", \"title\": \"Product\", \"description\": \"A product from Acme's catalog\", \"type\": \"object\", \"properties\": { \"id\": { \"description\": \"The unique identifier for a product\", \"type\": \"string\" }, \"time\": { \"description\": \"event timestamp of the event\", \"type\": \"string\", \"format\" : \"date-time\" } }, \"required\": [ \"id\", \"time\" ] }"); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof KafkaDaggerSource); @@ -124,7 +124,7 @@ public void shouldBeAbleToBuildAStreamWithFlinkKafkaConsumerDaggerSourceAndJsonS when(streamConfig.getSchemaTable()).thenReturn("data_stream"); when(streamConfig.getJsonSchema()).thenReturn("{ \"$schema\": \"https://json-schema.org/draft/2020-12/schema\", \"$id\": \"https://example.com/product.schema.json\", \"title\": \"Product\", \"description\": \"A product from Acme's catalog\", \"type\": \"object\", \"properties\": { \"id\": { \"description\": \"The unique identifier for a product\", \"type\": \"string\" }, \"time\": { \"description\": \"event timestamp of the event\", \"type\": \"string\", \"format\" : \"date-time\" } }, \"required\": [ \"id\", \"time\" ] }"); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof FlinkKafkaConsumerDaggerSource); @@ -139,7 +139,7 @@ public void shouldBeAbleToBuildAStreamWithParquetDaggerSourceAndProtoSchema() { when(streamConfig.getSchemaTable()).thenReturn("data_stream"); when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDClientSupplierMock); + Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplierMock); Stream stream = builder.build(); assertTrue(stream.getDaggerSource() instanceof ParquetDaggerSource); diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamsFactoryTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamsFactoryTest.java index dbc7be556..e54d152ab 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamsFactoryTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/StreamsFactoryTest.java @@ -1,9 +1,9 @@ package io.odpf.dagger.core.source; -import com.timgroup.statsd.StatsDClient; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.core.source.flinkkafkaconsumer.FlinkKafkaConsumerDaggerSource; import io.odpf.dagger.core.source.kafka.KafkaDaggerSource; +import io.odpf.depot.metrics.StatsDReporter; import io.odpf.stencil.client.StencilClient; import com.google.gson.JsonSyntaxException; import io.odpf.dagger.common.configuration.Configuration; @@ -34,7 +34,7 @@ public class StreamsFactoryTest { @Mock private Configuration configuration; - private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); @Before public void setup() { @@ -70,7 +70,7 @@ public void shouldReturnListOfStreamsCreatedFromConfiguration() { when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - List streams = StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDClientSupplierMock); + List streams = StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDReporterSupplierMock); assertEquals(2, streams.size()); assertTrue(streams.get(0).getDaggerSource() instanceof FlinkKafkaConsumerDaggerSource); @@ -93,7 +93,7 @@ public void shouldThrowErrorForInvalidStreamConfig() { when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); assertThrows(JsonSyntaxException.class, - () -> StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDClientSupplierMock)); + () -> StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDReporterSupplierMock)); } @Test @@ -101,6 +101,6 @@ public void shouldThrowNullPointerIfStreamConfigIsNotGiven() { when(configuration.getString(INPUT_STREAMS, "")).thenReturn(""); assertThrows(NullPointerException.class, - () -> StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDClientSupplierMock)); + () -> StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDReporterSupplierMock)); } } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java index 98476292f..b731dd96f 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java @@ -1,8 +1,7 @@ package io.odpf.dagger.core.source.parquet; -import com.timgroup.statsd.StatsDClient; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.common.serde.proto.deserialization.ProtoDeserializer; @@ -11,6 +10,7 @@ import io.odpf.dagger.core.source.config.models.SourceDetails; import io.odpf.dagger.core.source.config.models.SourceName; import io.odpf.dagger.core.source.config.models.SourceType; +import io.odpf.depot.metrics.StatsDReporter; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.connector.file.src.FileSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -48,7 +48,7 @@ public class ParquetDaggerSourceTest { @Mock private StreamExecutionEnvironment streamExecutionEnvironment; - private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); private FileSource fileSource; @@ -63,7 +63,7 @@ public void setup() { @Test public void shouldBeAbleToBuildSourceIfSourceDetailsIsBoundedParquetAndDaggerDeserializerIsSimpleGroupDeserializer() { when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDReporterSupplierMock); assertTrue(daggerSource.canBuild()); } @@ -72,7 +72,7 @@ public void shouldBeAbleToBuildSourceIfSourceDetailsIsBoundedParquetAndDaggerDes public void shouldNotBeAbleToBuildSourceIfSourceDetailsContainsMultipleBackToBackSources() { when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED), new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDReporterSupplierMock); assertFalse(daggerSource.canBuild()); } @@ -80,7 +80,7 @@ public void shouldNotBeAbleToBuildSourceIfSourceDetailsContainsMultipleBackToBac @Test public void shouldNotBeAbleToBuildSourceIfSourceNameIsUnsupported() { when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.KAFKA_CONSUMER, SourceType.BOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDReporterSupplierMock); assertFalse(daggerSource.canBuild()); } @@ -88,7 +88,7 @@ public void shouldNotBeAbleToBuildSourceIfSourceNameIsUnsupported() { @Test public void shouldNotBeAbleToBuildSourceIfSourceTypeIsUnsupported() { when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.UNBOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDReporterSupplierMock); assertFalse(daggerSource.canBuild()); } @@ -97,14 +97,14 @@ public void shouldNotBeAbleToBuildSourceIfSourceTypeIsUnsupported() { public void shouldNotBeAbleToBuildSourceIfDeserializerTypeIsUnsupported() { DaggerDeserializer unsupportedDeserializer = Mockito.mock(ProtoDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED)}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, unsupportedDeserializer, statsDClientSupplierMock); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, unsupportedDeserializer, statsDReporterSupplierMock); assertFalse(daggerSource.canBuild()); } @Test public void shouldBeAbleToRegisterSourceWithExecutionEnvironmentForCorrectConfiguration() { - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDReporterSupplierMock); ParquetDaggerSource daggerSourceSpy = Mockito.spy(daggerSource); doReturn(fileSource).when(daggerSourceSpy).buildFileSource(); when(streamConfig.getSchemaTable()).thenReturn("data_stream_0"); @@ -121,7 +121,7 @@ public void shouldUseStreamConfigurationToBuildTheFileSource() { when(streamConfig.getParquetFilesReadOrderStrategy()).thenReturn(EARLIEST_TIME_URL_FIRST); when(streamConfig.getParquetFilePaths()).thenReturn(new String[]{"gs://sshsh", "gs://shadd"}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDReporterSupplierMock); daggerSource.register(streamExecutionEnvironment, strategy); } @@ -131,7 +131,7 @@ public void shouldThrowRuntimeExceptionIfReadOrderStrategyIsNotSupported() { when(streamConfig.getParquetFilesReadOrderStrategy()).thenReturn(EARLIEST_INDEX_FIRST); when(streamConfig.getParquetFilePaths()).thenReturn(new String[]{"gs://sshsh", "gs://shadd"}); - ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDClientSupplierMock); + ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDReporterSupplierMock); assertThrows(DaggerConfigurationException.class, () -> daggerSource.register(streamExecutionEnvironment, strategy)); } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java index 6a8bb4f32..f87c87a28 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java @@ -1,8 +1,6 @@ package io.odpf.dagger.core.source.parquet; -import com.timgroup.statsd.StatsDClient; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; import io.odpf.dagger.core.source.parquet.reader.ParquetReader; import io.odpf.dagger.core.source.parquet.reader.ReaderProvider; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -17,7 +15,6 @@ import java.util.function.Supplier; import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; import static org.mockito.MockitoAnnotations.initMocks; public class ParquetFileRecordFormatTest { @@ -33,7 +30,6 @@ public class ParquetFileRecordFormatTest { private final ReaderProvider readerProviderMock = (filePath) -> parquetReader; private final Supplier> typeInformationProviderMock = () -> typeInformation; - private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); @Before public void setup() { @@ -45,7 +41,6 @@ public void shouldBuildAFileRecordFormatAsPerConfiguredParameters() { ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setParquetFileReaderProvider(readerProviderMock) .setTypeInformationProvider(typeInformationProviderMock) - .setStatsDClientSupplier(statsDClientSupplierMock) .build(); FileRecordFormat.Reader expectedReader = parquetFileRecordFormat.createReader(configuration, new Path("gs://file-path"), 0, 1024); @@ -62,7 +57,6 @@ public void shouldThrowIllegalArgumentExceptionWhenReaderProviderIsNotConfigured IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder .setTypeInformationProvider(typeInformationProviderMock) - .setStatsDClientSupplier(statsDClientSupplierMock) .build()); assertEquals("ReaderProvider is required but is set as null", ex.getMessage()); @@ -74,7 +68,6 @@ public void shouldThrowIllegalArgumentExceptionWhenTypeInformationProviderIsNotC IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder .setParquetFileReaderProvider(readerProviderMock) - .setStatsDClientSupplier(statsDClientSupplierMock) .build()); assertEquals("TypeInformationProvider is required but is set as null", ex.getMessage()); @@ -85,7 +78,6 @@ public void shouldReturnFalseWhenIsSplittableIsCalled() { ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setParquetFileReaderProvider(readerProviderMock) .setTypeInformationProvider(typeInformationProviderMock) - .setStatsDClientSupplier(statsDClientSupplierMock) .build(); assertFalse(parquetFileRecordFormat.isSplittable()); @@ -96,7 +88,6 @@ public void shouldThrowUnsupportedOperationExceptionWhenRestoreReaderIsCalled() ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setTypeInformationProvider(typeInformationProviderMock) .setParquetFileReaderProvider(readerProviderMock) - .setStatsDClientSupplier(statsDClientSupplierMock) .build(); UnsupportedOperationException ex = assertThrows(UnsupportedOperationException.class, diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java index 4110764d5..3145cc192 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java @@ -1,10 +1,10 @@ package io.odpf.dagger.core.source.parquet.reader; -import com.timgroup.statsd.StatsDClient; -import io.odpf.dagger.common.metrics.type.statsd.SerializedStatsDClientSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.ParquetFileSourceReaderInitializationException; +import io.odpf.depot.metrics.StatsDReporter; import org.apache.commons.lang3.ArrayUtils; import org.apache.flink.connector.file.src.util.CheckpointedPosition; import org.apache.flink.types.Row; @@ -40,7 +40,7 @@ public class ParquetReaderTest { @Rule public TemporaryFolder tempFolder = TemporaryFolder.builder().assureDeletion().build(); - private final SerializedStatsDClientSupplier statsDClientSupplierMock = () -> mock(StatsDClient.class); + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); @Before public void setup() { @@ -50,7 +50,7 @@ public void setup() { @Test public void shouldCreateReadersConfiguredWithTheSameDeserializerButForDifferentFilePaths() throws IOException { when(deserializer.deserialize(any(SimpleGroup.class))).thenReturn(Row.of("same", "deserializer")); - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); String filePath1 = classLoader.getResource("test_file.parquet").getPath(); @@ -64,7 +64,7 @@ public void shouldCreateReadersConfiguredWithTheSameDeserializerButForDifferentF @Test public void shouldReadFileAndCallDeserializerWithSimpleGroupWhenReadIsCalled() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); @@ -89,7 +89,7 @@ public void shouldReadFileAndCallDeserializerWithSimpleGroupWhenReadIsCalled() t @Test public void shouldBeAbleToReadParquetFileContainingMultipleRowGroups() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("multiple_row_groups_test_file.parquet").getPath()); @@ -117,7 +117,7 @@ public void shouldBeAbleToReadParquetFileContainingMultipleRowGroups() throws IO @Test public void shouldReturnDeserializedValueWhenRecordsPresentAndNullWhenNoMoreDataLeftToRead() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); when(deserializer.deserialize(any(SimpleGroup.class))).thenReturn(Row.of("some value")); @@ -130,7 +130,7 @@ public void shouldReturnDeserializedValueWhenRecordsPresentAndNullWhenNoMoreData @Test public void shouldThrowIOExceptionIfReadIsCalledAfterCallingClose() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); @@ -142,14 +142,14 @@ public void shouldThrowIOExceptionIfReadIsCalledAfterCallingClose() throws IOExc @Test public void shouldThrowParquetFileSourceReaderInitializationExceptionIfCannotConstructReaderForTheFile() throws IOException { final File tempFile = tempFolder.newFile("test_file.parquet"); - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); assertThrows(ParquetFileSourceReaderInitializationException.class, () -> provider.getReader(tempFile.getPath())); } @Test public void shouldReturnCheckPointedPositionWithNoOffsetAndZeroRecordsAfterOffsetWhenReadHasNotBeenCalledYet() { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); @@ -160,7 +160,7 @@ public void shouldReturnCheckPointedPositionWithNoOffsetAndZeroRecordsAfterOffse @Test public void shouldUpdateCheckPointedPositionWithNoOffsetAndCountOfTotalRecordsReadYet() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); @@ -173,7 +173,7 @@ public void shouldUpdateCheckPointedPositionWithNoOffsetAndCountOfTotalRecordsRe @Test public void shouldNotUpdateCheckpointedPositionWhenNoMoreRecordsToRead() throws IOException { - ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDClientSupplierMock); + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java index 52a652e20..b5255515b 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java @@ -7,7 +7,7 @@ import io.odpf.dagger.common.configuration.Configuration; import io.odpf.dagger.common.core.StreamInfo; import io.odpf.dagger.common.core.Transformer; -import io.odpf.dagger.common.metrics.type.flink.CounterStatsManager; +import io.odpf.dagger.common.metrics.manager.CounterStatsManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java index 0ad94d908..3800529af 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java @@ -1,6 +1,6 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.common.udfs.ScalarUdf; import io.odpf.dagger.functions.udfs.scalar.dart.DartAspects; import io.odpf.dagger.functions.udfs.scalar.dart.store.gcs.GcsDataStore; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java index 914b26ebf..4965d7828 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java @@ -1,6 +1,6 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.common.udfs.ScalarUdf; import io.odpf.dagger.functions.exceptions.KeyDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.DartAspects; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java index d9d935cf8..cc728a88b 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java @@ -6,7 +6,7 @@ import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; -import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; +import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; import io.odpf.dagger.functions.exceptions.BucketDoesNotExistException; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java index 9696b2c2b..df79b7380 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java @@ -1,7 +1,7 @@ package io.odpf.dagger.functions.udfs.scalar.dart.store.gcs; -import io.odpf.dagger.common.metrics.type.flink.GaugeStatsManager; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.functions.udfs.scalar.DartContains; import io.odpf.dagger.functions.udfs.scalar.DartGet; import io.odpf.dagger.functions.udfs.scalar.dart.DartAspects; diff --git a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java index 8ccb880c7..eace155fb 100644 --- a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java +++ b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java @@ -1,7 +1,7 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.store.gcs.GcsDataStore; import io.odpf.dagger.functions.udfs.scalar.dart.types.SetCache; diff --git a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java index 74747f4fd..1b7c9653d 100644 --- a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java +++ b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java @@ -1,6 +1,6 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.functions.exceptions.KeyDoesNotExistException; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.store.gcs.GcsDataStore; diff --git a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java index a864e3239..b832eb1ac 100644 --- a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java +++ b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java @@ -1,7 +1,7 @@ package io.odpf.dagger.functions.udfs.scalar.dart.store.gcs; -import io.odpf.dagger.common.metrics.type.flink.MeterStatsManager; +import io.odpf.dagger.common.metrics.manager.MeterStatsManager; import io.odpf.dagger.functions.exceptions.BucketDoesNotExistException; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.types.MapCache; From 1a7590ad0c5960afa72eec380421d0fb76ae9f7a Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 5 Jul 2022 14:24:59 +0530 Subject: [PATCH 11/47] feat: revert package name [https://github.com/odpf/dagger/issues/108] --- .../metrics/{manager => managers}/CounterStatsManager.java | 2 +- .../metrics/{manager => managers}/GaugeStatsManager.java | 2 +- .../metrics/{manager => managers}/MeterStatsManager.java | 2 +- .../main/java/io/odpf/dagger/common/udfs/AggregateUdf.java | 2 +- .../src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java | 2 +- .../src/main/java/io/odpf/dagger/common/udfs/TableUdf.java | 2 +- .../common/metrics/type/flink/CounterStatsManagerTest.java | 2 +- .../common/metrics/type/flink/GaugeStatsManagerTest.java | 2 +- .../common/metrics/type/flink/MeterStatsManagerTest.java | 2 +- .../odpf/dagger/core/processors/common/EndpointHandler.java | 2 +- .../dagger/core/processors/common/PostResponseTelemetry.java | 2 +- .../odpf/dagger/core/processors/external/AsyncConnector.java | 2 +- .../dagger/core/processors/external/es/EsAsyncConnector.java | 2 +- .../dagger/core/processors/external/es/EsResponseHandler.java | 2 +- .../core/processors/external/grpc/GrpcAsyncConnector.java | 2 +- .../core/processors/external/grpc/GrpcResponseHandler.java | 2 +- .../core/processors/external/http/HttpAsyncConnector.java | 2 +- .../core/processors/external/http/HttpResponseHandler.java | 2 +- .../dagger/core/processors/external/pg/PgAsyncConnector.java | 2 +- .../dagger/core/processors/external/pg/PgResponseHandler.java | 2 +- .../core/processors/longbow/processor/LongbowReader.java | 2 +- .../core/processors/longbow/processor/LongbowWriter.java | 2 +- .../telemetry/processor/MetricsTelemetryExporter.java | 2 +- .../dagger/core/processors/common/EndpointHandlerTest.java | 2 +- .../core/processors/external/es/EsAsyncConnectorTest.java | 2 +- .../core/processors/external/es/EsResponseHandlerTest.java | 2 +- .../core/processors/external/grpc/GrpcAsyncConnectorTest.java | 2 +- .../processors/external/grpc/GrpcResponseHandlerTest.java | 2 +- .../core/processors/external/http/HttpAsyncConnectorTest.java | 2 +- .../processors/external/http/HttpResponseHandlerTest.java | 2 +- .../core/processors/external/pg/PgAsyncConnectorTest.java | 2 +- .../core/processors/external/pg/PgResponseHandlerTest.java | 2 +- .../core/processors/longbow/processor/LongbowReaderTest.java | 2 +- .../core/processors/longbow/processor/LongbowWriterTest.java | 2 +- .../telemetry/processor/MetricsTelemetryExporterTest.java | 2 +- .../transformers/InvalidRecordFilterTransformer.java | 2 +- .../io/odpf/dagger/functions/udfs/scalar/DartContains.java | 2 +- .../java/io/odpf/dagger/functions/udfs/scalar/DartGet.java | 2 +- .../functions/udfs/scalar/dart/store/gcs/GcsClient.java | 2 +- .../functions/udfs/scalar/dart/store/gcs/GcsDataStore.java | 4 ++-- .../odpf/dagger/functions/udfs/scalar/DartContainsTest.java | 2 +- .../io/odpf/dagger/functions/udfs/scalar/DartGetTest.java | 2 +- .../udfs/scalar/dart/store/gcs/GcsDataStoreTest.java | 2 +- 43 files changed, 44 insertions(+), 44 deletions(-) rename dagger-common/src/main/java/io/odpf/dagger/common/metrics/{manager => managers}/CounterStatsManager.java (97%) rename dagger-common/src/main/java/io/odpf/dagger/common/metrics/{manager => managers}/GaugeStatsManager.java (98%) rename dagger-common/src/main/java/io/odpf/dagger/common/metrics/{manager => managers}/MeterStatsManager.java (98%) diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/CounterStatsManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/CounterStatsManager.java similarity index 97% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/CounterStatsManager.java rename to dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/CounterStatsManager.java index 3660ce000..8b924a1ae 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/CounterStatsManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/CounterStatsManager.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.manager; +package io.odpf.dagger.common.metrics.managers; import io.odpf.dagger.common.metrics.aspects.AspectType; import io.odpf.dagger.common.metrics.aspects.Aspects; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/GaugeStatsManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManager.java similarity index 98% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/GaugeStatsManager.java rename to dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManager.java index a57f34f77..79d8d32e4 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/GaugeStatsManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManager.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.manager; +package io.odpf.dagger.common.metrics.managers; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.MetricGroup; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/MeterStatsManager.java b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/MeterStatsManager.java similarity index 98% rename from dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/MeterStatsManager.java rename to dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/MeterStatsManager.java index 7b995eca1..46de410d7 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/metrics/manager/MeterStatsManager.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/metrics/managers/MeterStatsManager.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.manager; +package io.odpf.dagger.common.metrics.managers; import org.apache.flink.dropwizard.metrics.DropwizardHistogramWrapper; import org.apache.flink.dropwizard.metrics.DropwizardMeterWrapper; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java index 1a0596efd..e7242c8a9 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/AggregateUdf.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.udfs; -import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; +import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; import org.apache.flink.table.functions.AggregateFunction; import org.apache.flink.table.functions.FunctionContext; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java index f9ebde373..83685f25e 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/ScalarUdf.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.udfs; -import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; +import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.functions.ScalarFunction; diff --git a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java index 4c1eb054a..ae5b3f28e 100644 --- a/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java +++ b/dagger-common/src/main/java/io/odpf/dagger/common/udfs/TableUdf.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.udfs; -import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; +import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; import org.apache.flink.table.functions.FunctionContext; import org.apache.flink.table.functions.TableFunction; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java index ddfea637c..b12bb6c6f 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.metrics.type.flink; -import io.odpf.dagger.common.metrics.manager.CounterStatsManager; +import io.odpf.dagger.common.metrics.managers.CounterStatsManager; import io.odpf.dagger.common.metrics.utils.TestAspects; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.MetricGroup; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java index 3bcd4871c..b05739a02 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.metrics.type.flink; -import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; +import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; import io.odpf.dagger.common.metrics.utils.TestAspects; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.MetricGroup; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java index 4a6c5d796..c7f725c84 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java @@ -1,6 +1,6 @@ package io.odpf.dagger.common.metrics.type.flink; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.common.metrics.utils.TestAspects; import org.apache.flink.dropwizard.metrics.DropwizardMeterWrapper; import org.apache.flink.metrics.Histogram; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java index 274122962..22694b3c8 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/EndpointHandler.java @@ -7,7 +7,7 @@ import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; import io.odpf.dagger.core.exception.InvalidConfigurationException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.common.serde.typehandler.TypeHandler; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java index 76a0d6246..e8e479327 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/common/PostResponseTelemetry.java @@ -1,6 +1,6 @@ package io.odpf.dagger.core.processors.common; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import java.time.Instant; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java index cc6d2bbb2..eb5622e79 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/AsyncConnector.java @@ -8,7 +8,7 @@ import com.google.protobuf.Descriptors; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java index f29b39a3d..93e76a4bb 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnector.java @@ -1,6 +1,6 @@ package io.odpf.dagger.core.processors.external.es; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.common.PostResponseTelemetry; import io.odpf.dagger.core.processors.common.RowManager; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java index d0f81c416..09c0e613a 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/es/EsResponseHandler.java @@ -9,7 +9,7 @@ import com.jayway.jsonpath.PathNotFoundException; import io.odpf.dagger.common.serde.typehandler.TypeHandler; import io.odpf.dagger.common.serde.typehandler.TypeHandlerFactory; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.exception.HttpFailureException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java index 4a2a45398..173be65f2 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnector.java @@ -13,7 +13,7 @@ import io.odpf.dagger.core.exception.ChannelNotAvailableException; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; import io.odpf.dagger.core.exception.InvalidGrpcBodyException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.external.grpc.client.GrpcClient; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java index 4e1fa7e9d..12441ecb1 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandler.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.grpc; import io.odpf.dagger.core.exception.GrpcFailureException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.ColumnNameManager; import io.odpf.dagger.core.processors.common.OutputMapping; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java index 71c404b56..fcb401904 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnector.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.http; import io.odpf.dagger.core.exception.InvalidHttpVerbException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.common.DescriptorManager; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java index 89441b746..5b84c4803 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandler.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.http; import io.odpf.dagger.core.exception.HttpFailureException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.ColumnNameManager; import io.odpf.dagger.core.processors.common.OutputMapping; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java index 7c20ec595..46595164e 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnector.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.pg; import io.odpf.dagger.core.exception.InvalidConfigurationException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.common.PostResponseTelemetry; import io.odpf.dagger.core.processors.common.RowManager; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java index d94fc29b1..bcfd9cec8 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandler.java @@ -1,7 +1,7 @@ package io.odpf.dagger.core.processors.external.pg; import io.odpf.dagger.core.exception.HttpFailureException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.processors.ColumnNameManager; import io.odpf.dagger.core.processors.common.PostResponseTelemetry; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java index c10e0e52d..381abf73a 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReader.java @@ -5,7 +5,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.LongbowReaderAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.reporters.ErrorReporterFactory; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java index 0754df6fd..bd02ac9c9 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriter.java @@ -5,7 +5,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.LongbowWriterAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.reporters.ErrorReporterFactory; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java index adfac16e9..73f5c97c7 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporter.java @@ -6,7 +6,7 @@ import io.odpf.dagger.core.metrics.telemetry.TelemetryPublisher; import io.odpf.dagger.core.metrics.telemetry.TelemetrySubscriber; -import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; +import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; import io.odpf.dagger.core.metrics.aspects.TelemetryAspects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java index f338c5e14..e103815c6 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/common/EndpointHandlerTest.java @@ -4,7 +4,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.consumer.TestEnumType; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java index 2e6494fed..ade3dd94c 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsAsyncConnectorTest.java @@ -7,7 +7,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java index 3fa62417e..80b52ebcd 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/es/EsResponseHandlerTest.java @@ -2,7 +2,7 @@ import com.google.protobuf.Descriptors; import com.jayway.jsonpath.PathNotFoundException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestEnrichedBookingLogMessage; import io.odpf.dagger.consumer.TestProfile; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java index eaf1205ad..7b9c04857 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcAsyncConnectorTest.java @@ -4,7 +4,7 @@ import com.google.protobuf.DynamicMessage; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestGrpcRequest; import io.odpf.dagger.core.exception.InvalidConfigurationException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java index 6a4aa43c6..4218c95f6 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/grpc/GrpcResponseHandlerTest.java @@ -5,7 +5,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.jayway.jsonpath.PathNotFoundException; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestGrpcResponse; import io.odpf.dagger.consumer.TestLocation; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java index b10a51da8..2f401b59d 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpAsyncConnectorTest.java @@ -3,7 +3,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.exception.InvalidHttpVerbException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java index 68b44ef04..c5fc30f84 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/http/HttpResponseHandlerTest.java @@ -2,7 +2,7 @@ import com.google.protobuf.Descriptors; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestSurgeFactorLogMessage; import io.odpf.dagger.core.exception.HttpFailureException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java index 86b44f4c5..ec71872d5 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgAsyncConnectorTest.java @@ -3,7 +3,7 @@ import io.odpf.stencil.client.StencilClient; import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.exceptions.DescriptorNotFoundException; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.InvalidConfigurationException; import io.odpf.dagger.core.metrics.aspects.ExternalSourceAspects; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java index fe11b799e..693060719 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/external/pg/PgResponseHandlerTest.java @@ -2,7 +2,7 @@ import com.google.protobuf.Descriptors; import io.odpf.dagger.common.metrics.aspects.Aspects; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.consumer.TestSurgeFactorLogMessage; import io.odpf.dagger.core.exception.HttpFailureException; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java index 650a10945..ea59255cc 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowReaderTest.java @@ -4,7 +4,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.telemetry.TelemetrySubscriber; import io.odpf.dagger.core.processors.longbow.LongbowSchema; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java index f36eab349..da9fe73aa 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/longbow/processor/LongbowWriterTest.java @@ -5,7 +5,7 @@ import org.apache.flink.types.Row; import io.odpf.dagger.common.configuration.Configuration; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.core.metrics.aspects.LongbowWriterAspects; import io.odpf.dagger.core.metrics.reporters.ErrorReporter; import io.odpf.dagger.core.metrics.telemetry.TelemetrySubscriber; diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java index cf623430a..72cf3242e 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/processors/telemetry/processor/MetricsTelemetryExporterTest.java @@ -6,7 +6,7 @@ import org.apache.flink.metrics.groups.OperatorMetricGroup; import org.apache.flink.types.Row; -import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; +import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; import io.odpf.dagger.core.metrics.telemetry.TelemetryPublisher; import org.junit.Before; import org.junit.Test; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java index b5255515b..622b4e0c8 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/transformers/InvalidRecordFilterTransformer.java @@ -7,7 +7,7 @@ import io.odpf.dagger.common.configuration.Configuration; import io.odpf.dagger.common.core.StreamInfo; import io.odpf.dagger.common.core.Transformer; -import io.odpf.dagger.common.metrics.manager.CounterStatsManager; +import io.odpf.dagger.common.metrics.managers.CounterStatsManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java index 3800529af..ba817562f 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartContains.java @@ -1,6 +1,6 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.common.udfs.ScalarUdf; import io.odpf.dagger.functions.udfs.scalar.dart.DartAspects; import io.odpf.dagger.functions.udfs.scalar.dart.store.gcs.GcsDataStore; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java index 4965d7828..b35ddf4c0 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/DartGet.java @@ -1,6 +1,6 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.common.udfs.ScalarUdf; import io.odpf.dagger.functions.exceptions.KeyDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.DartAspects; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java index cc728a88b..208580f05 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsClient.java @@ -6,7 +6,7 @@ import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; -import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; +import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; import io.odpf.dagger.functions.exceptions.BucketDoesNotExistException; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; diff --git a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java index df79b7380..20648c2ba 100644 --- a/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java +++ b/dagger-functions/src/main/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStore.java @@ -1,7 +1,7 @@ package io.odpf.dagger.functions.udfs.scalar.dart.store.gcs; -import io.odpf.dagger.common.metrics.manager.GaugeStatsManager; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.functions.udfs.scalar.DartContains; import io.odpf.dagger.functions.udfs.scalar.DartGet; import io.odpf.dagger.functions.udfs.scalar.dart.DartAspects; diff --git a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java index eace155fb..3ea29af89 100644 --- a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java +++ b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartContainsTest.java @@ -1,7 +1,7 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.store.gcs.GcsDataStore; import io.odpf.dagger.functions.udfs.scalar.dart.types.SetCache; diff --git a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java index 1b7c9653d..ee6072ba5 100644 --- a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java +++ b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/DartGetTest.java @@ -1,6 +1,6 @@ package io.odpf.dagger.functions.udfs.scalar; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.functions.exceptions.KeyDoesNotExistException; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.store.gcs.GcsDataStore; diff --git a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java index b832eb1ac..95a4c5f5e 100644 --- a/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java +++ b/dagger-functions/src/test/java/io/odpf/dagger/functions/udfs/scalar/dart/store/gcs/GcsDataStoreTest.java @@ -1,7 +1,7 @@ package io.odpf.dagger.functions.udfs.scalar.dart.store.gcs; -import io.odpf.dagger.common.metrics.manager.MeterStatsManager; +import io.odpf.dagger.common.metrics.managers.MeterStatsManager; import io.odpf.dagger.functions.exceptions.BucketDoesNotExistException; import io.odpf.dagger.functions.exceptions.TagDoesNotExistException; import io.odpf.dagger.functions.udfs.scalar.dart.types.MapCache; From af18a80304349fcfc53db54d895c632bb83156a9 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 5 Jul 2022 14:33:02 +0530 Subject: [PATCH 12/47] feat: remove dogstatsd from dagger-common - revert package names refactoring [https://github.com/odpf/dagger/issues/108] --- dagger-common/build.gradle | 3 +-- .../{type/flink => managers}/CounterStatsManagerTest.java | 5 ++--- .../{type/flink => managers}/GaugeStatsManagerTest.java | 5 ++--- .../{type/flink => managers}/MeterStatsManagerTest.java | 5 ++--- .../common/metrics/{ => managers}/utils/TestAspects.java | 2 +- 5 files changed, 8 insertions(+), 12 deletions(-) rename dagger-common/src/test/java/io/odpf/dagger/common/metrics/{type/flink => managers}/CounterStatsManagerTest.java (94%) rename dagger-common/src/test/java/io/odpf/dagger/common/metrics/{type/flink => managers}/GaugeStatsManagerTest.java (92%) rename dagger-common/src/test/java/io/odpf/dagger/common/metrics/{type/flink => managers}/MeterStatsManagerTest.java (94%) rename dagger-common/src/test/java/io/odpf/dagger/common/metrics/{ => managers}/utils/TestAspects.java (92%) diff --git a/dagger-common/build.gradle b/dagger-common/build.gradle index 318450acc..4f74404f0 100644 --- a/dagger-common/build.gradle +++ b/dagger-common/build.gradle @@ -67,8 +67,7 @@ dependencies { dependenciesCommonJar 'org.apache.flink:flink-json:' + flinkVersion dependenciesCommonJar 'com.jayway.jsonpath:json-path:2.4.0' dependenciesCommonJar 'io.odpf:stencil:0.1.6' - dependenciesCommonJar 'com.datadoghq:java-dogstatsd-client:4.0.0' - dependenciesCommonJar 'com.google.code.gson:gson:2.8.2' + dependenciesCommonJar 'com.google.code.gson:gson:2.8.2' dependenciesCommonJar 'org.apache.parquet:parquet-column:1.12.2' testImplementation 'junit:junit:4.13' diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/CounterStatsManagerTest.java similarity index 94% rename from dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java rename to dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/CounterStatsManagerTest.java index b12bb6c6f..4a96300d9 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/CounterStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/CounterStatsManagerTest.java @@ -1,7 +1,6 @@ -package io.odpf.dagger.common.metrics.type.flink; +package io.odpf.dagger.common.metrics.managers; -import io.odpf.dagger.common.metrics.managers.CounterStatsManager; -import io.odpf.dagger.common.metrics.utils.TestAspects; +import io.odpf.dagger.common.metrics.managers.utils.TestAspects; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.metrics.SimpleCounter; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManagerTest.java similarity index 92% rename from dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java rename to dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManagerTest.java index b05739a02..e88013116 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/GaugeStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/GaugeStatsManagerTest.java @@ -1,7 +1,6 @@ -package io.odpf.dagger.common.metrics.type.flink; +package io.odpf.dagger.common.metrics.managers; -import io.odpf.dagger.common.metrics.managers.GaugeStatsManager; -import io.odpf.dagger.common.metrics.utils.TestAspects; +import io.odpf.dagger.common.metrics.managers.utils.TestAspects; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.MetricGroup; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java similarity index 94% rename from dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java rename to dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java index c7f725c84..a675334a5 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/type/flink/MeterStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java @@ -1,7 +1,6 @@ -package io.odpf.dagger.common.metrics.type.flink; +package io.odpf.dagger.common.metrics.managers; -import io.odpf.dagger.common.metrics.managers.MeterStatsManager; -import io.odpf.dagger.common.metrics.utils.TestAspects; +import io.odpf.dagger.common.metrics.managers.utils.TestAspects; import org.apache.flink.dropwizard.metrics.DropwizardMeterWrapper; import org.apache.flink.metrics.Histogram; import org.apache.flink.metrics.Meter; diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/utils/TestAspects.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/utils/TestAspects.java similarity index 92% rename from dagger-common/src/test/java/io/odpf/dagger/common/metrics/utils/TestAspects.java rename to dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/utils/TestAspects.java index 7234bf8b2..bc2e02d2e 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/utils/TestAspects.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/utils/TestAspects.java @@ -1,4 +1,4 @@ -package io.odpf.dagger.common.metrics.utils; +package io.odpf.dagger.common.metrics.managers.utils; import io.odpf.dagger.common.metrics.aspects.AspectType; import io.odpf.dagger.common.metrics.aspects.Aspects; From 13ace7637c76b5b1f08ad87f61a76027144bad3e Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 5 Jul 2022 14:34:52 +0530 Subject: [PATCH 13/47] feat: fix indentation [https://github.com/odpf/dagger/issues/108] --- dagger-common/build.gradle | 2 +- .../dagger/common/metrics/managers/MeterStatsManagerTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dagger-common/build.gradle b/dagger-common/build.gradle index 4f74404f0..a41ff942f 100644 --- a/dagger-common/build.gradle +++ b/dagger-common/build.gradle @@ -67,7 +67,7 @@ dependencies { dependenciesCommonJar 'org.apache.flink:flink-json:' + flinkVersion dependenciesCommonJar 'com.jayway.jsonpath:json-path:2.4.0' dependenciesCommonJar 'io.odpf:stencil:0.1.6' - dependenciesCommonJar 'com.google.code.gson:gson:2.8.2' + dependenciesCommonJar 'com.google.code.gson:gson:2.8.2' dependenciesCommonJar 'org.apache.parquet:parquet-column:1.12.2' testImplementation 'junit:junit:4.13' diff --git a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java index a675334a5..5a5c4134e 100644 --- a/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java +++ b/dagger-common/src/test/java/io/odpf/dagger/common/metrics/managers/MeterStatsManagerTest.java @@ -1,11 +1,11 @@ package io.odpf.dagger.common.metrics.managers; -import io.odpf.dagger.common.metrics.managers.utils.TestAspects; import org.apache.flink.dropwizard.metrics.DropwizardMeterWrapper; import org.apache.flink.metrics.Histogram; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MetricGroup; +import io.odpf.dagger.common.metrics.managers.utils.TestAspects; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; From 5f081091c46883df2c03f93945dd60cc03a069f3 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Tue, 5 Jul 2022 19:42:56 +0530 Subject: [PATCH 14/47] feat: create inner class for provider - return empty string in getMetricStatsDTags [https://github.com/odpf/dagger/issues/108] --- .../io/odpf/dagger/core/StreamManager.java | 7 ++-- .../reporters/statsd/DaggerMetricsConfig.java | 2 +- ...rovider.java => DaggerStatsDReporter.java} | 38 ++++++++++++------- 3 files changed, 28 insertions(+), 19 deletions(-) rename dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/{DaggerStatsDReporterProvider.java => DaggerStatsDReporter.java} (57%) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java index 8bd482639..606de25ef 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/StreamManager.java @@ -1,7 +1,6 @@ package io.odpf.dagger.core; -import io.odpf.dagger.core.metrics.reporters.statsd.DaggerStatsDReporterProvider; -import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.DaggerStatsDReporter; import io.odpf.dagger.core.source.Stream; import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.datastream.DataStream; @@ -223,7 +222,7 @@ private void addSink(StreamInfo streamInfo) { List getStreams() { org.apache.flink.configuration.Configuration flinkConfiguration = (org.apache.flink.configuration.Configuration) this.executionEnvironment.getConfiguration(); - SerializedStatsDReporterSupplier statsDReporterSupplier = DaggerStatsDReporterProvider.provide(flinkConfiguration, configuration); - return StreamsFactory.getStreams(configuration, stencilClientOrchestrator, statsDReporterSupplier); + DaggerStatsDReporter daggerStatsDReporter = DaggerStatsDReporter.Provider.provide(flinkConfiguration, configuration); + return StreamsFactory.getStreams(configuration, stencilClientOrchestrator, daggerStatsDReporter); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfig.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfig.java index aec1209fa..15ea4ffed 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfig.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfig.java @@ -38,6 +38,6 @@ public Integer getMetricStatsDPort() { @Override public String getMetricStatsDTags() { - return null; + return ""; } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterProvider.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java similarity index 57% rename from dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterProvider.java rename to dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java index 60ee35c58..edf09978b 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterProvider.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java @@ -1,23 +1,36 @@ package io.odpf.dagger.core.metrics.reporters.statsd; -import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; import io.odpf.dagger.core.metrics.reporters.statsd.tags.GlobalTags; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; import io.odpf.depot.metrics.StatsDReporter; import io.odpf.depot.metrics.StatsDReporterBuilder; import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import static io.odpf.dagger.core.utils.Constants.FLINK_JOB_ID_DEFAULT; import static io.odpf.dagger.core.utils.Constants.FLINK_JOB_ID_KEY; -public class DaggerStatsDReporterProvider { +public class DaggerStatsDReporter implements SerializedStatsDReporterSupplier { private static StatsDReporter statsDReporter; - private static final Logger LOGGER = LoggerFactory.getLogger(DaggerStatsDReporterProvider.class.getName()); + private final Configuration flinkConfiguration; + private final io.odpf.dagger.common.configuration.Configuration daggerConfiguration; - public static SerializedStatsDReporterSupplier provide(Configuration flinkConfiguration, io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { + private DaggerStatsDReporter(Configuration flinkConfiguration, io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { + this.flinkConfiguration = flinkConfiguration; + this.daggerConfiguration = daggerConfiguration; + } + + private String[] generateGlobalTags(io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { + StatsDTag[] globalTags = new StatsDTag[]{ + new StatsDTag(GlobalTags.JOB_ID, daggerConfiguration.getString(FLINK_JOB_ID_KEY, FLINK_JOB_ID_DEFAULT))}; + return Arrays.stream(globalTags) + .map(StatsDTag::getFormattedTag) + .toArray(String[]::new); + } + + @Override + public StatsDReporter getStatsDReporter() { if (statsDReporter == null) { DaggerMetricsConfig daggerMetricsConfig = new DaggerMetricsConfig(flinkConfiguration); String[] globalTags = generateGlobalTags(daggerConfiguration); @@ -26,16 +39,13 @@ public static SerializedStatsDReporterSupplier provide(Configuration flinkConfig .withMetricConfig(daggerMetricsConfig) .withExtraTags(globalTags) .build(); - LOGGER.info("Instantiated new StatsDReporter"); } - return () -> statsDReporter; + return statsDReporter; } - private static String[] generateGlobalTags(io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { - StatsDTag[] globalTags = new StatsDTag[]{ - new StatsDTag(GlobalTags.JOB_ID, daggerConfiguration.getString(FLINK_JOB_ID_KEY, FLINK_JOB_ID_DEFAULT))}; - return Arrays.stream(globalTags) - .map(StatsDTag::getFormattedTag) - .toArray(String[]::new); + public static class Provider { + public static DaggerStatsDReporter provide(Configuration flinkConfiguration, io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { + return new DaggerStatsDReporter(flinkConfiguration, daggerConfiguration); + } } } From 92b36bb19a5ff9523350cf4ec1231e2ddcb4ea72 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Wed, 6 Jul 2022 13:24:51 +0530 Subject: [PATCH 15/47] feat: use statsd.increment method instead of captureCount [https://github.com/odpf/dagger/issues/108] --- .../reporters/statsd/manager/DaggerCounterManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java index c85771c3d..53b728479 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java @@ -37,7 +37,7 @@ public void increment(Aspects aspect) { @Override public void increment(Aspects aspect, long positiveCount) { - statsDReporter.captureCount(aspect.getValue(), positiveCount, formattedTags); + statsDReporter.getClient().count(aspect.getValue(), positiveCount, formattedTags); } @Override @@ -47,6 +47,6 @@ public void decrement(Aspects aspect) { @Override public void decrement(Aspects aspect, long negativeCount) { - statsDReporter.captureCount(aspect.getValue(), negativeCount, formattedTags); + statsDReporter.getClient().count(aspect.getValue(), negativeCount, formattedTags); } } From 53f7cc3668536ec7f9b1e3979b6be4764fbcbfe9 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Wed, 6 Jul 2022 14:22:09 +0530 Subject: [PATCH 16/47] feat: use statsdReporter directly instead of client - global tags fail to be sent if statsdClient is used - remove unused interface methods [https://github.com/odpf/dagger/issues/108] --- .../statsd/manager/DaggerCounterManager.java | 4 ++-- .../statsd/manager/DaggerGaugeManager.java | 15 ++------------- .../statsd/manager/DaggerHistogramManager.java | 7 +------ .../statsd/manager/MeasurementManager.java | 1 + .../reporters/statsd/measurement/Gauge.java | 6 +----- .../reporters/statsd/measurement/Histogram.java | 2 -- 6 files changed, 7 insertions(+), 28 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java index 53b728479..c85771c3d 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java @@ -37,7 +37,7 @@ public void increment(Aspects aspect) { @Override public void increment(Aspects aspect, long positiveCount) { - statsDReporter.getClient().count(aspect.getValue(), positiveCount, formattedTags); + statsDReporter.captureCount(aspect.getValue(), positiveCount, formattedTags); } @Override @@ -47,6 +47,6 @@ public void decrement(Aspects aspect) { @Override public void decrement(Aspects aspect, long negativeCount) { - statsDReporter.getClient().count(aspect.getValue(), negativeCount, formattedTags); + statsDReporter.captureCount(aspect.getValue(), negativeCount, formattedTags); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java index bcadd7f85..c13e35ff6 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java @@ -8,7 +8,6 @@ import java.util.ArrayList; - public class DaggerGaugeManager implements MeasurementManager, Gauge { private final StatsDReporter statsDReporter; private String[] formattedTags; @@ -32,17 +31,7 @@ public void register(StatsDTag[] tags) { } @Override - public void registerLong(Aspects aspect, long gaugeValue) { - statsDReporter.getClient().gauge(aspect.getValue(), gaugeValue, formattedTags); - } - - @Override - public void registerString(Aspects aspect, String gaugeValue) { - throw new UnsupportedOperationException("DogStatsD library doesn't support reporting non-numeric values as gauge values."); - } - - @Override - public void registerDouble(Aspects aspect, double gaugeValue) { - statsDReporter.getClient().gauge(aspect.getValue(), gaugeValue, formattedTags); + public void registerInteger(Aspects aspect, int gaugeValue) { + statsDReporter.gauge(aspect.getValue(), gaugeValue, formattedTags); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java index b5f6e233a..b0530afa8 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java @@ -32,11 +32,6 @@ public void register(StatsDTag[] tags) { @Override public void recordValue(Aspects aspect, long value) { - statsDReporter.getClient().histogram(aspect.getValue(), value, formattedTags); - } - - @Override - public void recordValue(Aspects aspect, double value) { - statsDReporter.getClient().histogram(aspect.getValue(), value, formattedTags); + statsDReporter.captureHistogram(aspect.getValue(), value, formattedTags); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java index ee46fa3d6..bdef1896b 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java @@ -7,5 +7,6 @@ public interface MeasurementManager extends Serializable { void register(Aspects[] aspect, StatsDTag[] tags); + void register(StatsDTag[] tags); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java index 8c50edaf4..a6ac906f5 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java @@ -5,9 +5,5 @@ import java.io.Serializable; public interface Gauge extends Serializable { - void registerLong(Aspects aspect, long gaugeValue); - - void registerString(Aspects aspect, String gaugeValue); - - void registerDouble(Aspects aspect, double gaugeValue); + void registerInteger(Aspects aspect, int gaugeValue); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Histogram.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Histogram.java index 338f20d1d..1c8f68cfe 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Histogram.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Histogram.java @@ -6,6 +6,4 @@ public interface Histogram extends Serializable { void recordValue(Aspects aspect, long value); - - void recordValue(Aspects aspect, double value); } From 72da9a5cff9cc0f0ee8faa76461a801aae2e6315 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Wed, 6 Jul 2022 15:11:33 +0530 Subject: [PATCH 17/47] feat: use = as tag separator [https://github.com/odpf/dagger/issues/108] --- .../dagger/core/metrics/reporters/statsd/tags/StatsDTag.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java index 1b2cbefb5..bec416416 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java @@ -28,7 +28,7 @@ public String getFormattedTag() { if (tagValue.equals(NIL_TAG_VALUE)) { return tagKey; } else { - return String.format("%s:%s", tagKey, tagValue); + return String.format("%s=%s", tagKey, tagValue); } } } From 0f32e00b3f1d9333a6886155ab67377f335df96b Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Wed, 6 Jul 2022 15:12:13 +0530 Subject: [PATCH 18/47] feat: use = as tag separator [https://github.com/odpf/dagger/issues/108] --- .../reporters/statsd/manager/DaggerGaugeManager.java | 2 +- .../core/metrics/reporters/statsd/measurement/Gauge.java | 2 +- .../splitassigner/ChronologyOrderedSplitAssigner.java | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java index c13e35ff6..8793f830a 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java @@ -31,7 +31,7 @@ public void register(StatsDTag[] tags) { } @Override - public void registerInteger(Aspects aspect, int gaugeValue) { + public void markValue(Aspects aspect, int gaugeValue) { statsDReporter.gauge(aspect.getValue(), gaugeValue, formattedTags); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java index a6ac906f5..a41fb29bd 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Gauge.java @@ -5,5 +5,5 @@ import java.io.Serializable; public interface Gauge extends Serializable { - void registerInteger(Aspects aspect, int gaugeValue); + void markValue(Aspects aspect, int gaugeValue); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java index 9d9f00674..32822d0df 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java @@ -34,15 +34,15 @@ public class ChronologyOrderedSplitAssigner implements FileSplitAssigner { private ChronologyOrderedSplitAssigner(Collection fileSourceSplits, PathParser pathParser, TimeRangePool timeRangePool, SerializedStatsDReporterSupplier statsDReporterSupplier) { this.registerTagsWithMeasurementManagers(statsDReporterSupplier); - daggerGaugeManager.registerLong(TOTAL_SPLITS_DISCOVERED, fileSourceSplits.size()); + daggerGaugeManager.markValue(TOTAL_SPLITS_DISCOVERED, fileSourceSplits.size()); this.pathParser = pathParser; this.timeRangePool = timeRangePool; this.unassignedSplits = new PriorityBlockingQueue<>(INITIAL_DEFAULT_CAPACITY, getFileSourceSplitComparator()); for (FileSourceSplit split : fileSourceSplits) { validateAndAddSplits(split); } - daggerGaugeManager.registerLong(TOTAL_SPLITS_RECORDED, unassignedSplits.size()); - daggerGaugeManager.registerLong(SPLITS_AWAITING_ASSIGNMENT, unassignedSplits.size()); + daggerGaugeManager.markValue(TOTAL_SPLITS_RECORDED, unassignedSplits.size()); + daggerGaugeManager.markValue(SPLITS_AWAITING_ASSIGNMENT, unassignedSplits.size()); } private void registerTagsWithMeasurementManagers(SerializedStatsDReporterSupplier statsDReporterSupplier) { @@ -57,7 +57,7 @@ public Optional getNext(@Nullable String hostname) { if (instantEnrichedSplit == null) { return Optional.empty(); } - daggerGaugeManager.registerLong(SPLITS_AWAITING_ASSIGNMENT, unassignedSplits.size()); + daggerGaugeManager.markValue(SPLITS_AWAITING_ASSIGNMENT, unassignedSplits.size()); return Optional.of(instantEnrichedSplit.getFileSourceSplit()); } From ec6bd630e62da65a814f9049be2e607383f8e8ac Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Thu, 7 Jul 2022 12:53:21 +0530 Subject: [PATCH 19/47] feat: delete dagger source tag [https://github.com/odpf/dagger/issues/108] --- .../core/metrics/reporters/statsd/tags/ComponentTags.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/ComponentTags.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/ComponentTags.java index 39d156f5d..b73d64c1b 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/ComponentTags.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/ComponentTags.java @@ -4,13 +4,11 @@ public class ComponentTags { private static StatsDTag[] parquetReaderTags; private static StatsDTag[] splitAssignerTags; private static final String COMPONENT_TAG_KEY = "component"; - private static final String DAGGER_SOURCE_TAG_KEY = "dagger_source"; public static StatsDTag[] getParquetReaderTags() { if (parquetReaderTags == null) { parquetReaderTags = new StatsDTag[]{ new StatsDTag(COMPONENT_TAG_KEY, "parquet_reader"), - new StatsDTag(DAGGER_SOURCE_TAG_KEY, "parquet_source") }; } return parquetReaderTags; @@ -20,7 +18,6 @@ public static StatsDTag[] getSplitAssignerTags() { if (splitAssignerTags == null) { splitAssignerTags = new StatsDTag[]{ new StatsDTag(COMPONENT_TAG_KEY, "split_assigner"), - new StatsDTag(DAGGER_SOURCE_TAG_KEY, "parquet_source") }; } return splitAssignerTags; From 20b607b92c571f0489d4a62942a81eb0248c152f Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Thu, 7 Jul 2022 13:00:49 +0530 Subject: [PATCH 20/47] feat: delete unused register method from interface [https://github.com/odpf/dagger/issues/108] --- .../reporters/statsd/manager/DaggerCounterManager.java | 5 ----- .../metrics/reporters/statsd/manager/DaggerGaugeManager.java | 5 ----- .../reporters/statsd/manager/DaggerHistogramManager.java | 5 ----- .../metrics/reporters/statsd/manager/MeasurementManager.java | 3 --- 4 files changed, 18 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java index c85771c3d..3c7579598 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java @@ -16,11 +16,6 @@ public DaggerCounterManager(SerializedStatsDReporterSupplier statsDReporterSuppl this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); } - @Override - public void register(Aspects[] aspect, StatsDTag[] tags) { - register(tags); - } - @Override public void register(StatsDTag[] tags) { ArrayList tagList = new ArrayList<>(); diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java index 8793f830a..004e5180f 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java @@ -16,11 +16,6 @@ public DaggerGaugeManager(SerializedStatsDReporterSupplier statsDReporterSupplie this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); } - @Override - public void register(Aspects[] aspect, StatsDTag[] tags) { - register(tags); - } - @Override public void register(StatsDTag[] tags) { ArrayList tagList = new ArrayList<>(); diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java index b0530afa8..0beebca72 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java @@ -16,11 +16,6 @@ public DaggerHistogramManager(SerializedStatsDReporterSupplier statsDReporterSup this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); } - @Override - public void register(Aspects[] aspect, StatsDTag[] tags) { - register(tags); - } - @Override public void register(StatsDTag[] tags) { ArrayList tagList = new ArrayList<>(); diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java index bdef1896b..5c195c17b 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java @@ -1,12 +1,9 @@ package io.odpf.dagger.core.metrics.reporters.statsd.manager; -import io.odpf.dagger.common.metrics.aspects.Aspects; import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; import java.io.Serializable; public interface MeasurementManager extends Serializable { - void register(Aspects[] aspect, StatsDTag[] tags); - void register(StatsDTag[] tags); } From cfd01d6f7166885355b5574a9a8c2991a1293e1e Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Thu, 7 Jul 2022 13:48:18 +0530 Subject: [PATCH 21/47] feat: add StatsDErrorReporter [https://github.com/odpf/dagger/issues/108] --- .../reporters/statsd/StatsDErrorReporter.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java new file mode 100644 index 000000000..7bf14e1e6 --- /dev/null +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java @@ -0,0 +1,35 @@ +package io.odpf.dagger.core.metrics.reporters.statsd; + +import io.odpf.dagger.core.metrics.reporters.ErrorReporter; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; +import io.odpf.depot.metrics.StatsDReporter; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; + +import static io.odpf.dagger.core.utils.Constants.FATAL_EXCEPTION_METRIC_GROUP_KEY; +import static io.odpf.dagger.core.utils.Constants.NONFATAL_EXCEPTION_METRIC_GROUP_KEY; + +public class StatsDErrorReporter implements ErrorReporter { + private static final String fatalExceptionTag = new StatsDTag(FATAL_EXCEPTION_METRIC_GROUP_KEY).getFormattedTag(); + private static final String nonFatalExceptionTag = new StatsDTag(NONFATAL_EXCEPTION_METRIC_GROUP_KEY).getFormattedTag(); + private final StatsDReporter statsDReporter; + + public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSupplier) { + this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); + } + + @Override + public void reportFatalException(Exception exception) { + statsDReporter.captureCount(exception.getClass().getName(), 1L, fatalExceptionTag); + } + + @Override + public void reportNonFatalException(Exception exception) { + statsDReporter.captureCount(exception.getClass().getName(), 1L, nonFatalExceptionTag); + } + + @Override + public Counter addExceptionToCounter(Exception exception, MetricGroup metricGroup, String metricGroupKey) { + throw new UnsupportedOperationException("This operation is not supported on StatsDReporter"); + } +} From 94383b8f2ed1065a2a70aba87adecc9e0001d2e6 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Thu, 7 Jul 2022 14:08:23 +0530 Subject: [PATCH 22/47] feat: add exception message to measurement as well [https://github.com/odpf/dagger/issues/108] --- .../core/metrics/reporters/statsd/StatsDErrorReporter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java index 7bf14e1e6..21a24abdf 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java @@ -12,6 +12,8 @@ public class StatsDErrorReporter implements ErrorReporter { private static final String fatalExceptionTag = new StatsDTag(FATAL_EXCEPTION_METRIC_GROUP_KEY).getFormattedTag(); private static final String nonFatalExceptionTag = new StatsDTag(NONFATAL_EXCEPTION_METRIC_GROUP_KEY).getFormattedTag(); + private static final String FATAL_EXCEPTION_EVENT_MEASUREMENT = "fatal_exception_events"; + private static final String NON_FATAL_EXCEPTION_EVENT_MEASUREMENT = "non_fatal_exception_events"; private final StatsDReporter statsDReporter; public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSupplier) { @@ -21,11 +23,13 @@ public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSuppli @Override public void reportFatalException(Exception exception) { statsDReporter.captureCount(exception.getClass().getName(), 1L, fatalExceptionTag); + statsDReporter.recordEvent(FATAL_EXCEPTION_EVENT_MEASUREMENT, exception.getMessage(), fatalExceptionTag); } @Override public void reportNonFatalException(Exception exception) { statsDReporter.captureCount(exception.getClass().getName(), 1L, nonFatalExceptionTag); + statsDReporter.recordEvent(NON_FATAL_EXCEPTION_EVENT_MEASUREMENT, exception.getMessage(), nonFatalExceptionTag); } @Override From 217828119d6b4a248944ab4ec7d96915bbb0b774 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Thu, 7 Jul 2022 14:56:07 +0530 Subject: [PATCH 23/47] feat: remove exception message measurement [https://github.com/odpf/dagger/issues/108] --- .../core/metrics/reporters/statsd/StatsDErrorReporter.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java index 21a24abdf..7bf14e1e6 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java @@ -12,8 +12,6 @@ public class StatsDErrorReporter implements ErrorReporter { private static final String fatalExceptionTag = new StatsDTag(FATAL_EXCEPTION_METRIC_GROUP_KEY).getFormattedTag(); private static final String nonFatalExceptionTag = new StatsDTag(NONFATAL_EXCEPTION_METRIC_GROUP_KEY).getFormattedTag(); - private static final String FATAL_EXCEPTION_EVENT_MEASUREMENT = "fatal_exception_events"; - private static final String NON_FATAL_EXCEPTION_EVENT_MEASUREMENT = "non_fatal_exception_events"; private final StatsDReporter statsDReporter; public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSupplier) { @@ -23,13 +21,11 @@ public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSuppli @Override public void reportFatalException(Exception exception) { statsDReporter.captureCount(exception.getClass().getName(), 1L, fatalExceptionTag); - statsDReporter.recordEvent(FATAL_EXCEPTION_EVENT_MEASUREMENT, exception.getMessage(), fatalExceptionTag); } @Override public void reportNonFatalException(Exception exception) { statsDReporter.captureCount(exception.getClass().getName(), 1L, nonFatalExceptionTag); - statsDReporter.recordEvent(NON_FATAL_EXCEPTION_EVENT_MEASUREMENT, exception.getMessage(), nonFatalExceptionTag); } @Override From 4ff6048a9826eb3dd37de611553f960e7645c56f Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Thu, 7 Jul 2022 17:46:55 +0530 Subject: [PATCH 24/47] feat: report errors with exception class as tag value [https://github.com/odpf/dagger/issues/108] --- .../metrics/reporters/statsd/StatsDErrorReporter.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java index 7bf14e1e6..610058933 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java @@ -10,8 +10,8 @@ import static io.odpf.dagger.core.utils.Constants.NONFATAL_EXCEPTION_METRIC_GROUP_KEY; public class StatsDErrorReporter implements ErrorReporter { - private static final String fatalExceptionTag = new StatsDTag(FATAL_EXCEPTION_METRIC_GROUP_KEY).getFormattedTag(); - private static final String nonFatalExceptionTag = new StatsDTag(NONFATAL_EXCEPTION_METRIC_GROUP_KEY).getFormattedTag(); + private static final String fatalExceptionTagKey = "fatal_exception_type"; + private static final String nonFatalExceptionTagKey = "non_fatal_exception_type"; private final StatsDReporter statsDReporter; public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSupplier) { @@ -20,12 +20,14 @@ public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSuppli @Override public void reportFatalException(Exception exception) { - statsDReporter.captureCount(exception.getClass().getName(), 1L, fatalExceptionTag); + StatsDTag statsDTag = new StatsDTag(fatalExceptionTagKey, exception.getClass().getName()); + statsDReporter.captureCount(FATAL_EXCEPTION_METRIC_GROUP_KEY, 1L, statsDTag.getFormattedTag()); } @Override public void reportNonFatalException(Exception exception) { - statsDReporter.captureCount(exception.getClass().getName(), 1L, nonFatalExceptionTag); + StatsDTag statsDTag = new StatsDTag(nonFatalExceptionTagKey, exception.getClass().getName()); + statsDReporter.captureCount(NONFATAL_EXCEPTION_METRIC_GROUP_KEY, 1L, statsDTag.getFormattedTag()); } @Override From 4aaafb2cf68cbf5b84c774074891713e1dae9dc8 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Thu, 7 Jul 2022 17:48:42 +0530 Subject: [PATCH 25/47] feat: capture and report fatal exceptions [https://github.com/odpf/dagger/issues/108] --- .../DaggerDeserializerFactory.java | 11 +++++-- .../core/source/DaggerSourceFactory.java | 7 +++-- .../io/odpf/dagger/core/source/Stream.java | 2 +- .../source/parquet/ParquetDaggerSource.java | 12 +++++-- .../parquet/ParquetFileRecordFormat.java | 31 ++++++++++++++++--- .../source/parquet/ParquetFileSource.java | 22 +++++++++++-- .../source/parquet/reader/ParquetReader.java | 21 +++++++++++-- .../ChronologyOrderedSplitAssigner.java | 19 +++++++++--- 8 files changed, 103 insertions(+), 22 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/deserializer/DaggerDeserializerFactory.java b/dagger-core/src/main/java/io/odpf/dagger/core/deserializer/DaggerDeserializerFactory.java index 2c8e2a954..3955cfbb9 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/deserializer/DaggerDeserializerFactory.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/deserializer/DaggerDeserializerFactory.java @@ -4,6 +4,8 @@ import io.odpf.dagger.common.core.StencilClientOrchestrator; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.core.exception.DaggerConfigurationException; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.StatsDErrorReporter; import io.odpf.dagger.core.source.config.StreamConfig; import org.apache.flink.types.Row; @@ -12,12 +14,17 @@ import java.util.stream.Stream; public class DaggerDeserializerFactory { - public static DaggerDeserializer create(StreamConfig streamConfig, Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator) { + public static DaggerDeserializer create(StreamConfig streamConfig, Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator, SerializedStatsDReporterSupplier statsDReporterSupplier) { return getDaggerDeserializerProviders(streamConfig, configuration, stencilClientOrchestrator) .stream() .filter(DaggerDeserializerProvider::canProvide) .findFirst() - .orElseThrow(() -> new DaggerConfigurationException("No suitable deserializer could be constructed for the given stream configuration.")) + .orElseThrow(() -> { + StatsDErrorReporter statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); + DaggerConfigurationException ex = new DaggerConfigurationException("No suitable deserializer could be constructed for the given stream configuration."); + statsDErrorReporter.reportFatalException(ex); + return ex; + }) .getDaggerDeserializer(); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java index 231bf9b93..b99735342 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/DaggerSourceFactory.java @@ -4,6 +4,7 @@ import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.core.exception.InvalidDaggerSourceException; +import io.odpf.dagger.core.metrics.reporters.statsd.StatsDErrorReporter; import io.odpf.dagger.core.source.config.StreamConfig; import io.odpf.dagger.core.source.flinkkafkaconsumer.FlinkKafkaConsumerDaggerSource; import io.odpf.dagger.core.source.kafka.KafkaDaggerSource; @@ -23,9 +24,11 @@ public static DaggerSource create(StreamConfig streamConfig, Configuration .filter(DaggerSource::canBuild) .findFirst() .orElseThrow(() -> { + StatsDErrorReporter statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); String sourceDetails = Arrays.toString(streamConfig.getSourceDetails()); - String message = String.format("No suitable DaggerSource can be created as per SOURCE_DETAILS config %s", sourceDetails); - return new InvalidDaggerSourceException(message); + InvalidDaggerSourceException ex = new InvalidDaggerSourceException(String.format("No suitable DaggerSource can be created as per SOURCE_DETAILS config %s", sourceDetails)); + statsDErrorReporter.reportFatalException(ex); + return ex; }); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java index 638b7f2b4..3f5266b0b 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/Stream.java @@ -43,7 +43,7 @@ public Builder(StreamConfig streamConfig, Configuration configuration, StencilCl } public Stream build() { - DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator); + DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplier); DaggerSource daggerSource = DaggerSourceFactory.create(streamConfig, configuration, daggerDeserializer, statsDReporterSupplier); return new Stream(daggerSource, streamConfig.getSchemaTable()); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java index 2f55876da..4ef5cc4d5 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java @@ -5,6 +5,7 @@ import io.odpf.dagger.common.serde.DaggerDeserializer; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.DaggerConfigurationException; +import io.odpf.dagger.core.metrics.reporters.statsd.StatsDErrorReporter; import io.odpf.dagger.core.source.DaggerSource; import io.odpf.dagger.core.source.config.StreamConfig; import io.odpf.dagger.core.source.config.models.SourceDetails; @@ -37,12 +38,14 @@ public class ParquetDaggerSource implements DaggerSource { private final SerializedStatsDReporterSupplier statsDReporterSupplier; private static final SourceType SUPPORTED_SOURCE_TYPE = BOUNDED; private static final SourceName SUPPORTED_SOURCE_NAME = PARQUET_SOURCE; + private final StatsDErrorReporter statsDErrorReporter; public ParquetDaggerSource(StreamConfig streamConfig, Configuration configuration, DaggerDeserializer deserializer, SerializedStatsDReporterSupplier statsDReporterSupplier) { this.streamConfig = streamConfig; this.configuration = configuration; this.deserializer = deserializer; this.statsDReporterSupplier = statsDReporterSupplier; + this.statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); } @Override @@ -74,6 +77,7 @@ FileSource buildFileSource() { .setFileRecordFormat(parquetFileRecordFormat) .setSourceType(SUPPORTED_SOURCE_TYPE) .setFileSplitAssigner(splitAssignerProvider) + .setStatsDReporterSupplier(statsDReporterSupplier) .build(); return parquetFileSource.buildFileSource(); } @@ -96,8 +100,11 @@ private FileSplitAssigner.Provider buildParquetFileSplitAssignerProvider() { .addPathParser(new HourDatePathParser()); return chronologyOrderedSplitAssignerBuilder::build; case EARLIEST_INDEX_FIRST: - default: - throw new DaggerConfigurationException("Error: file split assignment strategy not configured or not supported yet."); + default: { + DaggerConfigurationException daggerConfigurationException = new DaggerConfigurationException("Error: file split assignment strategy not configured or not supported yet."); + statsDErrorReporter.reportFatalException(daggerConfigurationException); + throw daggerConfigurationException; + } } } @@ -109,6 +116,7 @@ private ParquetFileRecordFormat buildParquetFileRecordFormat() { return parquetFileRecordFormatBuilder .setParquetFileReaderProvider(parquetFileReaderProvider) .setTypeInformationProvider(typeInformationProvider) + .setStatsDReporterSupplier(statsDReporterSupplier) .build(); } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java index 9828dfabb..77a5fd84a 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java @@ -2,6 +2,8 @@ import static com.google.api.client.util.Preconditions.checkArgument; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.StatsDErrorReporter; import io.odpf.dagger.core.source.parquet.reader.ReaderProvider; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; @@ -14,10 +16,12 @@ public class ParquetFileRecordFormat implements FileRecordFormat { private final ReaderProvider parquetFileReaderProvider; private final Supplier> typeInformationProvider; + private final StatsDErrorReporter statsDErrorReporter; - private ParquetFileRecordFormat(ReaderProvider parquetFileReaderProvider, Supplier> typeInformationProvider) { + private ParquetFileRecordFormat(ReaderProvider parquetFileReaderProvider, Supplier> typeInformationProvider, SerializedStatsDReporterSupplier statsDReporterSupplier) { this.parquetFileReaderProvider = parquetFileReaderProvider; this.typeInformationProvider = typeInformationProvider; + this.statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); } @Override @@ -27,8 +31,10 @@ public Reader createReader(Configuration config, Path filePath, long splitO @Override public Reader restoreReader(Configuration config, Path filePath, long restoredOffset, long splitOffset, long splitLength) { - throw new UnsupportedOperationException("Error: ParquetReader do not have offsets and hence cannot be restored " + UnsupportedOperationException ex = new UnsupportedOperationException("Error: ParquetReader do not have offsets and hence cannot be restored " + "via this method."); + statsDErrorReporter.reportFatalException(ex); + throw ex; } @Override @@ -44,6 +50,7 @@ public TypeInformation getProducedType() { public static class Builder { private ReaderProvider parquetFileReaderProvider; private Supplier> typeInformationProvider; + private SerializedStatsDReporterSupplier statsDReporterSupplier; public static Builder getInstance() { return new Builder(); @@ -52,6 +59,7 @@ public static Builder getInstance() { private Builder() { this.parquetFileReaderProvider = null; this.typeInformationProvider = null; + this.statsDReporterSupplier = null; } public Builder setParquetFileReaderProvider(ReaderProvider parquetFileReaderProvider) { @@ -64,10 +72,23 @@ public Builder setTypeInformationProvider(Supplier> typeInf return this; } + public Builder setStatsDReporterSupplier(SerializedStatsDReporterSupplier statsDReporterSupplier) { + this.statsDReporterSupplier = statsDReporterSupplier; + return this; + } + public ParquetFileRecordFormat build() { - checkArgument(parquetFileReaderProvider != null, "ReaderProvider is required but is set as null"); - checkArgument(typeInformationProvider != null, "TypeInformationProvider is required but is set as null"); - return new ParquetFileRecordFormat(parquetFileReaderProvider, typeInformationProvider); + try { + checkArgument(parquetFileReaderProvider != null, "ReaderProvider is required but is set as null"); + checkArgument(typeInformationProvider != null, "TypeInformationProvider is required but is set as null"); + checkArgument(statsDReporterSupplier != null, "SerializedStatsDReporterSupplier is required but is set as null"); + return new ParquetFileRecordFormat(parquetFileReaderProvider, typeInformationProvider, statsDReporterSupplier); + } catch (IllegalArgumentException ex) { + if (statsDReporterSupplier != null) { + new StatsDErrorReporter(statsDReporterSupplier).reportFatalException(ex); + } + throw ex; + } } } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileSource.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileSource.java index 56ceebe81..f97fb14c1 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileSource.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileSource.java @@ -1,6 +1,8 @@ package io.odpf.dagger.core.source.parquet; import io.odpf.dagger.common.configuration.Configuration; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.StatsDErrorReporter; import io.odpf.dagger.core.source.config.models.SourceType; import lombok.Getter; import org.apache.flink.connector.file.src.FileSource; @@ -51,6 +53,7 @@ public static class Builder { private FileRecordFormat fileRecordFormat; private Configuration configuration; private FileSplitAssigner.Provider fileSplitAssigner; + private SerializedStatsDReporterSupplier statsDReporterSupplier; public static Builder getInstance() { return new Builder(); @@ -89,12 +92,25 @@ public Builder setConfiguration(Configuration configuration) { return this; } + public Builder setStatsDReporterSupplier(SerializedStatsDReporterSupplier statsDReporterSupplier) { + this.statsDReporterSupplier = statsDReporterSupplier; + return this; + } + /* other validations if required before creating the file source can be put here */ /* for example, checking that all the file paths conform to just one partitioning strategy */ private void sanityCheck() { - checkArgument(fileRecordFormat != null, "FileRecordFormat is required but is set as null"); - checkArgument(filePaths.length != 0, "At least one file path is required but none are provided"); - checkArgument(sourceType == BOUNDED, "Running Parquet FileSource in UNBOUNDED mode is not supported yet"); + try { + checkArgument(statsDReporterSupplier != null, "SerializedStatsDReporterSupplier is required but is set as null"); + checkArgument(fileRecordFormat != null, "FileRecordFormat is required but is set as null"); + checkArgument(filePaths.length != 0, "At least one file path is required but none are provided"); + checkArgument(sourceType == BOUNDED, "Running Parquet FileSource in UNBOUNDED mode is not supported yet"); + } catch (IllegalArgumentException exception) { + if (statsDReporterSupplier != null) { + new StatsDErrorReporter(statsDReporterSupplier).reportFatalException(exception); + } + throw exception; + } } public ParquetFileSource build() { diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java index 139b40f9e..017aa92ba 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/reader/ParquetReader.java @@ -1,6 +1,8 @@ package io.odpf.dagger.core.source.parquet.reader; +import io.odpf.dagger.common.exceptions.serde.DaggerDeserializationException; import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.StatsDErrorReporter; import io.odpf.dagger.core.metrics.reporters.statsd.manager.DaggerCounterManager; import io.odpf.dagger.core.metrics.reporters.statsd.manager.DaggerHistogramManager; import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; @@ -43,6 +45,7 @@ public class ParquetReader implements FileRecordFormat.Reader { private long totalEmittedRowCount; private DaggerCounterManager daggerCounterManager; private DaggerHistogramManager daggerHistogramManager; + private final StatsDErrorReporter statsDErrorReporter; private static final Logger LOGGER = LoggerFactory.getLogger(ParquetReader.class.getName()); private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDeserializer, ParquetFileReader @@ -54,6 +57,7 @@ private ParquetReader(Path hadoopFilePath, SimpleGroupDeserializer simpleGroupDe this.isRecordReaderInitialized = false; this.totalEmittedRowCount = 0L; this.registerTagsWithMeasurementManagers(statsDReporterSupplier); + this.statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); daggerCounterManager.increment(ParquetReaderAspects.READER_CREATED); } @@ -104,7 +108,9 @@ private Row readRecords() throws IOException { currentRecordIndex++; long startDeserializationTime = Instant.now().toEpochMilli(); - Row row = simpleGroupDeserializer.deserialize(simpleGroup); + + Row row = deserialize(simpleGroup); + long endDeserializationTime = Instant.now().toEpochMilli(); totalEmittedRowCount++; @@ -113,6 +119,15 @@ private Row readRecords() throws IOException { return row; } + private Row deserialize(SimpleGroup simpleGroup) { + try { + return simpleGroupDeserializer.deserialize(simpleGroup); + } catch (DaggerDeserializationException exception) { + statsDErrorReporter.reportFatalException(exception); + throw exception; + } + } + @Nullable @Override public Row read() throws IOException { @@ -162,7 +177,9 @@ public ParquetReader getReader(String filePath) { ParquetFileReader parquetFileReader = ParquetFileReader.open(HadoopInputFile.fromPath(hadoopFilePath, conf)); return new ParquetReader(hadoopFilePath, simpleGroupDeserializer, parquetFileReader, statsDReporterSupplier); } catch (IOException | RuntimeException ex) { - throw new ParquetFileSourceReaderInitializationException(ex); + ParquetFileSourceReaderInitializationException exception = new ParquetFileSourceReaderInitializationException(ex); + new StatsDErrorReporter(statsDReporterSupplier).reportFatalException(exception); + throw exception; } } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java index 32822d0df..9bf0f8fac 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java @@ -1,6 +1,7 @@ package io.odpf.dagger.core.source.parquet.splitassigner; import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.StatsDErrorReporter; import io.odpf.dagger.core.metrics.reporters.statsd.manager.DaggerGaugeManager; import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; import io.odpf.dagger.core.exception.PathParserNotProvidedException; @@ -19,6 +20,7 @@ import java.util.concurrent.PriorityBlockingQueue; import java.util.stream.Collectors; +import static com.google.api.client.util.Preconditions.checkArgument; import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.SPLITS_AWAITING_ASSIGNMENT; import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.TOTAL_SPLITS_DISCOVERED; import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.TOTAL_SPLITS_RECORDED; @@ -30,6 +32,7 @@ public class ChronologyOrderedSplitAssigner implements FileSplitAssigner { private PathParser pathParser; private TimeRangePool timeRangePool; private DaggerGaugeManager daggerGaugeManager; + private final StatsDErrorReporter statsDErrorReporter; private ChronologyOrderedSplitAssigner(Collection fileSourceSplits, PathParser pathParser, TimeRangePool timeRangePool, SerializedStatsDReporterSupplier statsDReporterSupplier) { @@ -37,6 +40,7 @@ private ChronologyOrderedSplitAssigner(Collection fileSourceSpl daggerGaugeManager.markValue(TOTAL_SPLITS_DISCOVERED, fileSourceSplits.size()); this.pathParser = pathParser; this.timeRangePool = timeRangePool; + statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); this.unassignedSplits = new PriorityBlockingQueue<>(INITIAL_DEFAULT_CAPACITY, getFileSourceSplitComparator()); for (FileSourceSplit split : fileSourceSplits) { validateAndAddSplits(split); @@ -83,7 +87,9 @@ private void validateAndAddSplits(FileSourceSplit split) { this.unassignedSplits.add(new InstantEnrichedSplit(split, instant)); } } catch (ParseException ex) { - throw new IllegalArgumentException(ex); + IllegalArgumentException exception = new IllegalArgumentException(ex); + statsDErrorReporter.reportFatalException(exception); + throw exception; } } @@ -104,7 +110,7 @@ private Comparator getFileSourceSplitComparator() { public static class ChronologyOrderedSplitAssignerBuilder implements Serializable { private PathParser pathParser; private TimeRangePool parquetFileDateRange; - private SerializedStatsDReporterSupplier supplier; + private SerializedStatsDReporterSupplier statsDReporterSupplier; public ChronologyOrderedSplitAssignerBuilder addPathParser(PathParser parser) { this.pathParser = parser; @@ -117,15 +123,18 @@ public ChronologyOrderedSplitAssignerBuilder addTimeRanges(TimeRangePool timeRan } public ChronologyOrderedSplitAssignerBuilder addStatsDReporterSupplier(SerializedStatsDReporterSupplier statsDReporterSupplier) { - this.supplier = statsDReporterSupplier; + this.statsDReporterSupplier = statsDReporterSupplier; return this; } public ChronologyOrderedSplitAssigner build(Collection fileSourceSplits) { + checkArgument(statsDReporterSupplier != null, "SerializedStatsDReporterSupplier is required but is set as null"); if (pathParser == null) { - throw new PathParserNotProvidedException("Path parser is null"); + PathParserNotProvidedException exception = new PathParserNotProvidedException("Path parser is null"); + new StatsDErrorReporter(statsDReporterSupplier).reportFatalException(exception); + throw exception; } - return new ChronologyOrderedSplitAssigner(fileSourceSplits, pathParser, parquetFileDateRange, supplier); + return new ChronologyOrderedSplitAssigner(fileSourceSplits, pathParser, parquetFileDateRange, statsDReporterSupplier); } } } From a724c9d9e6e533b164a79909b71cb3f90dffd862 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Thu, 7 Jul 2022 18:40:45 +0530 Subject: [PATCH 26/47] feat: fix broken tests and checkstyle issues [https://github.com/odpf/dagger/issues/108] --- .../statsd/DaggerStatsDReporter.java | 4 +- .../reporters/statsd/StatsDErrorReporter.java | 8 +-- .../source/parquet/ParquetDaggerSource.java | 3 +- .../ChronologyOrderedSplitAssigner.java | 4 +- .../DaggerDeserializerFactoryTest.java | 13 +++-- .../parquet/ParquetFileRecordFormatTest.java | 9 ++++ .../source/parquet/ParquetFileSourceTest.java | 15 +++++- .../ChronologyOrderedSplitAssignerTest.java | 52 +++++++++++++++---- 8 files changed, 83 insertions(+), 25 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java index edf09978b..5dae82ad0 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java @@ -21,7 +21,7 @@ private DaggerStatsDReporter(Configuration flinkConfiguration, io.odpf.dagger.co this.daggerConfiguration = daggerConfiguration; } - private String[] generateGlobalTags(io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { + private String[] generateGlobalTags() { StatsDTag[] globalTags = new StatsDTag[]{ new StatsDTag(GlobalTags.JOB_ID, daggerConfiguration.getString(FLINK_JOB_ID_KEY, FLINK_JOB_ID_DEFAULT))}; return Arrays.stream(globalTags) @@ -33,7 +33,7 @@ private String[] generateGlobalTags(io.odpf.dagger.common.configuration.Configur public StatsDReporter getStatsDReporter() { if (statsDReporter == null) { DaggerMetricsConfig daggerMetricsConfig = new DaggerMetricsConfig(flinkConfiguration); - String[] globalTags = generateGlobalTags(daggerConfiguration); + String[] globalTags = generateGlobalTags(); statsDReporter = StatsDReporterBuilder .builder() .withMetricConfig(daggerMetricsConfig) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java index 610058933..682f8dbad 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java @@ -10,8 +10,8 @@ import static io.odpf.dagger.core.utils.Constants.NONFATAL_EXCEPTION_METRIC_GROUP_KEY; public class StatsDErrorReporter implements ErrorReporter { - private static final String fatalExceptionTagKey = "fatal_exception_type"; - private static final String nonFatalExceptionTagKey = "non_fatal_exception_type"; + private static final String FATAL_EXCEPTION_TAG_KEY = "fatal_exception_type"; + private static final String NON_FATAL_EXCEPTION_TAG_KEY = "non_fatal_exception_type"; private final StatsDReporter statsDReporter; public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSupplier) { @@ -20,13 +20,13 @@ public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSuppli @Override public void reportFatalException(Exception exception) { - StatsDTag statsDTag = new StatsDTag(fatalExceptionTagKey, exception.getClass().getName()); + StatsDTag statsDTag = new StatsDTag(FATAL_EXCEPTION_TAG_KEY, exception.getClass().getName()); statsDReporter.captureCount(FATAL_EXCEPTION_METRIC_GROUP_KEY, 1L, statsDTag.getFormattedTag()); } @Override public void reportNonFatalException(Exception exception) { - StatsDTag statsDTag = new StatsDTag(nonFatalExceptionTagKey, exception.getClass().getName()); + StatsDTag statsDTag = new StatsDTag(NON_FATAL_EXCEPTION_TAG_KEY, exception.getClass().getName()); statsDReporter.captureCount(NONFATAL_EXCEPTION_METRIC_GROUP_KEY, 1L, statsDTag.getFormattedTag()); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java index 4ef5cc4d5..078b59731 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSource.java @@ -100,11 +100,10 @@ private FileSplitAssigner.Provider buildParquetFileSplitAssignerProvider() { .addPathParser(new HourDatePathParser()); return chronologyOrderedSplitAssignerBuilder::build; case EARLIEST_INDEX_FIRST: - default: { + default: DaggerConfigurationException daggerConfigurationException = new DaggerConfigurationException("Error: file split assignment strategy not configured or not supported yet."); statsDErrorReporter.reportFatalException(daggerConfigurationException); throw daggerConfigurationException; - } } } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java index 9bf0f8fac..b219ccc88 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssigner.java @@ -122,8 +122,8 @@ public ChronologyOrderedSplitAssignerBuilder addTimeRanges(TimeRangePool timeRan return this; } - public ChronologyOrderedSplitAssignerBuilder addStatsDReporterSupplier(SerializedStatsDReporterSupplier statsDReporterSupplier) { - this.statsDReporterSupplier = statsDReporterSupplier; + public ChronologyOrderedSplitAssignerBuilder addStatsDReporterSupplier(SerializedStatsDReporterSupplier supplier) { + this.statsDReporterSupplier = supplier; return this; } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/deserializer/DaggerDeserializerFactoryTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/deserializer/DaggerDeserializerFactoryTest.java index ded8de2a1..0b7144f88 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/deserializer/DaggerDeserializerFactoryTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/deserializer/DaggerDeserializerFactoryTest.java @@ -8,10 +8,12 @@ import io.odpf.dagger.common.serde.proto.deserialization.ProtoDeserializer; import io.odpf.dagger.consumer.TestBookingLogMessage; import io.odpf.dagger.core.exception.DaggerConfigurationException; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.core.source.config.models.SourceDetails; import io.odpf.dagger.core.source.config.models.SourceName; import io.odpf.dagger.core.source.config.models.SourceType; import io.odpf.dagger.core.source.config.StreamConfig; +import io.odpf.depot.metrics.StatsDReporter; import io.odpf.stencil.client.StencilClient; import org.apache.flink.types.Row; import org.junit.Before; @@ -19,6 +21,7 @@ import org.mockito.Mock; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -36,6 +39,8 @@ public class DaggerDeserializerFactoryTest { @Mock private StencilClient stencilClient; + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); + @Before public void setUp() throws Exception { initMocks(this); @@ -47,7 +52,7 @@ public void shouldReturnJsonDeserializerWhenConfigured() { when(streamConfig.getDataType()).thenReturn("JSON"); when(streamConfig.getJsonSchema()).thenReturn("{ \"$schema\": \"https://json-schema.org/draft/2020-12/schema\", \"$id\": \"https://example.com/product.schema.json\", \"title\": \"Product\", \"description\": \"A product from Acme's catalog\", \"type\": \"object\", \"properties\": { \"id\": { \"description\": \"The unique identifier for a product\", \"type\": \"string\" }, \"time\": { \"description\": \"event timestamp of the event\", \"type\": \"string\", \"format\" : \"date-time\" } }, \"required\": [ \"id\", \"time\" ] }"); - DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator); + DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplierMock); assertTrue(daggerDeserializer instanceof JsonDeserializer); } @@ -61,7 +66,7 @@ public void shouldReturnProtoDeserializerWhenConfigured() { when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator); + DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplierMock); assertTrue(daggerDeserializer instanceof ProtoDeserializer); } @@ -75,7 +80,7 @@ public void shouldReturnSimpleGroupDeserializerWhenConfigured() { when(stencilClientOrchestrator.getStencilClient()).thenReturn(stencilClient); when(stencilClient.get("com.tests.TestMessage")).thenReturn(TestBookingLogMessage.getDescriptor()); - DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator); + DaggerDeserializer daggerDeserializer = DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplierMock); assertTrue(daggerDeserializer instanceof SimpleGroupDeserializer); } @@ -85,6 +90,6 @@ public void shouldThrowRuntimeExceptionIfNoDeserializerCouldBeCreatedFromConfigs when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.BOUNDED)}); when(streamConfig.getDataType()).thenReturn("JSON"); - assertThrows(DaggerConfigurationException.class, () -> DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator)); + assertThrows(DaggerConfigurationException.class, () -> DaggerDeserializerFactory.create(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplierMock)); } } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java index f87c87a28..19577921a 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java @@ -1,8 +1,10 @@ package io.odpf.dagger.core.source.parquet; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.core.source.parquet.reader.ParquetReader; import io.odpf.dagger.core.source.parquet.reader.ReaderProvider; +import io.odpf.depot.metrics.StatsDReporter; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.file.src.reader.FileRecordFormat; @@ -15,6 +17,7 @@ import java.util.function.Supplier; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; import static org.mockito.MockitoAnnotations.initMocks; public class ParquetFileRecordFormatTest { @@ -30,6 +33,7 @@ public class ParquetFileRecordFormatTest { private final ReaderProvider readerProviderMock = (filePath) -> parquetReader; private final Supplier> typeInformationProviderMock = () -> typeInformation; + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); @Before public void setup() { @@ -41,6 +45,7 @@ public void shouldBuildAFileRecordFormatAsPerConfiguredParameters() { ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setParquetFileReaderProvider(readerProviderMock) .setTypeInformationProvider(typeInformationProviderMock) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .build(); FileRecordFormat.Reader expectedReader = parquetFileRecordFormat.createReader(configuration, new Path("gs://file-path"), 0, 1024); @@ -57,6 +62,7 @@ public void shouldThrowIllegalArgumentExceptionWhenReaderProviderIsNotConfigured IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder .setTypeInformationProvider(typeInformationProviderMock) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .build()); assertEquals("ReaderProvider is required but is set as null", ex.getMessage()); @@ -68,6 +74,7 @@ public void shouldThrowIllegalArgumentExceptionWhenTypeInformationProviderIsNotC IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder .setParquetFileReaderProvider(readerProviderMock) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .build()); assertEquals("TypeInformationProvider is required but is set as null", ex.getMessage()); @@ -78,6 +85,7 @@ public void shouldReturnFalseWhenIsSplittableIsCalled() { ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setParquetFileReaderProvider(readerProviderMock) .setTypeInformationProvider(typeInformationProviderMock) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .build(); assertFalse(parquetFileRecordFormat.isSplittable()); @@ -88,6 +96,7 @@ public void shouldThrowUnsupportedOperationExceptionWhenRestoreReaderIsCalled() ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setTypeInformationProvider(typeInformationProviderMock) .setParquetFileReaderProvider(readerProviderMock) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .build(); UnsupportedOperationException ex = assertThrows(UnsupportedOperationException.class, diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileSourceTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileSourceTest.java index c38482478..ed2bd5ed0 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileSourceTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileSourceTest.java @@ -1,10 +1,12 @@ package io.odpf.dagger.core.source.parquet; import io.odpf.dagger.common.configuration.Configuration; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.core.source.config.models.SourceType; import io.odpf.dagger.core.source.parquet.ParquetFileSource.Builder; import io.odpf.dagger.core.source.parquet.path.HourDatePathParser; import io.odpf.dagger.core.source.parquet.splitassigner.ChronologyOrderedSplitAssigner; +import io.odpf.depot.metrics.StatsDReporter; import org.apache.flink.connector.file.src.assigners.LocalityAwareSplitAssigner; import org.apache.flink.connector.file.src.reader.FileRecordFormat; import org.apache.flink.core.fs.Path; @@ -15,6 +17,7 @@ import static java.util.Collections.emptyList; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; import static org.mockito.MockitoAnnotations.initMocks; public class ParquetFileSourceTest { @@ -24,6 +27,8 @@ public class ParquetFileSourceTest { @Mock private FileRecordFormat fileRecordFormat; + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); + @Before public void setup() { initMocks(this); @@ -34,11 +39,14 @@ public void shouldBuildParquetFileSourceAsPerArguments() { Builder builder = Builder.getInstance(); Path[] filePaths = new Path[]{new Path("gs://aadadc"), new Path("gs://sjsjhd")}; ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); - splitAssignerBuilder.addPathParser(new HourDatePathParser()); + splitAssignerBuilder + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .addPathParser(new HourDatePathParser()); ParquetFileSource parquetFileSource = builder.setConfiguration(configuration) .setFileRecordFormat(fileRecordFormat) .setSourceType(SourceType.BOUNDED) .setFileSplitAssigner(splitAssignerBuilder::build) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .setFilePaths(filePaths) .build(); @@ -58,6 +66,7 @@ public void shouldThrowExceptionIfSourceTypeConfiguredAsUnbounded() { () -> builder.setConfiguration(configuration) .setFileRecordFormat(fileRecordFormat) .setSourceType(SourceType.UNBOUNDED) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .setFileSplitAssigner(new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder()::build) .setFilePaths(filePaths) .build()); @@ -73,6 +82,7 @@ public void shouldThrowExceptionIfFileRecordFormatIsNotSet() { IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder.setConfiguration(configuration) .setSourceType(SourceType.UNBOUNDED) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .setFileSplitAssigner(new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder()::build) .setFilePaths(filePaths) .build()); @@ -88,6 +98,7 @@ public void shouldThrowExceptionIfNoFilePathsSet() { () -> builder.setConfiguration(configuration) .setFileRecordFormat(fileRecordFormat) .setSourceType(SourceType.BOUNDED) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .setFileSplitAssigner(new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder()::build) .build()); @@ -100,6 +111,7 @@ public void shouldUseDefaultValueForSomeFieldsWhichAreNotConfiguredExplicitly() Path[] filePaths = new Path[]{new Path("gs://aadadc"), new Path("gs://sjsjhd")}; ParquetFileSource parquetFileSource = builder.setConfiguration(configuration) .setFileRecordFormat(fileRecordFormat) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .setFilePaths(filePaths) .build(); @@ -118,6 +130,7 @@ public void shouldReturnAFileSourceMadeFromParquetFileSource() { .setFileRecordFormat(fileRecordFormat) .setSourceType(SourceType.BOUNDED) .setFileSplitAssigner(new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder()::build) + .setStatsDReporterSupplier(statsDReporterSupplierMock) .setFilePaths(filePaths) .build(); diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssignerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssignerTest.java index c6252c23d..09fa72d4f 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssignerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssignerTest.java @@ -2,9 +2,11 @@ import io.odpf.dagger.core.exception.PathParserNotProvidedException; +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.core.source.config.models.TimeRange; import io.odpf.dagger.core.source.config.models.TimeRangePool; import io.odpf.dagger.core.source.parquet.path.HourDatePathParser; +import io.odpf.depot.metrics.StatsDReporter; import org.apache.flink.connector.file.src.FileSourceSplit; import org.apache.flink.core.fs.Path; import org.junit.Assert; @@ -19,11 +21,14 @@ import java.util.Optional; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; public class ChronologyOrderedSplitAssignerTest { + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); + @Test public void shouldReturnFileSplitsHavingOldestDateFilePathsFirstWhenFilePathURLHasOnlyDate() { FileSourceSplit firstSplit = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2019-10-12/asdghsdhasd"), 0, 1024); @@ -35,7 +40,9 @@ public void shouldReturnFileSplitsHavingOldestDateFilePathsFirstWhenFilePathURLH ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); - ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder.addPathParser(new HourDatePathParser()).build(inputSplits); + ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder.addPathParser(new HourDatePathParser()) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(inputSplits); for (int i = 0; i < 4; i++) { Optional split = splitAssigner.getNext(null); @@ -55,7 +62,9 @@ public void shouldReturnFileSplitsHavingOldestTimeFilePathsFirstWhenFilePathURLH ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); - ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder.addPathParser(new HourDatePathParser()).build(inputSplits); + ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder.addPathParser(new HourDatePathParser()) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(inputSplits); for (int i = 0; i < 4; i++) { Optional split = splitAssigner.getNext(null); @@ -71,7 +80,10 @@ public void shouldReturnEmptyOptionalWhenNoMoreSplitsToReturn() { ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); - ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder.addPathParser(new HourDatePathParser()).build(inputSplits); + ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder. + addPathParser(new HourDatePathParser()) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(inputSplits); splitAssigner.getNext(null); Optional nextSplit = splitAssigner.getNext(null); @@ -86,7 +98,10 @@ public void shouldThrowIllegalArgumentExceptionDuringConstructionItselfWhenFileP IllegalArgumentException ex = Assert.assertThrows(IllegalArgumentException.class, () -> { ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); - splitAssignerBuilder.addPathParser(new HourDatePathParser()).build(Collections.singleton(split)); + splitAssignerBuilder + .addPathParser(new HourDatePathParser()) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(Collections.singleton(split)); }); assertEquals("java.text.ParseException: Cannot extract timestamp from filepath for deciding order of processing.\n" @@ -101,7 +116,10 @@ public void shouldAddNewFileSourceSplitsWithOldestDateFilePathsReturnedFirstWhen FileSourceSplit fourthSplit = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2020-12-31/hagga6a36dg"), 0, 1024); ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); - ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder.addPathParser(new HourDatePathParser()).build(Collections.singleton(secondSplit)); + ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder + .addPathParser(new HourDatePathParser()) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(Collections.singleton(secondSplit)); List remainingSplitsToAdd = Arrays.asList(fourthSplit, firstSplit, thirdSplit); splitAssigner.addSplits(remainingSplitsToAdd); @@ -121,7 +139,10 @@ public void shouldAddNewFileSourceSplitsWithOldestTimeFilePathsReturnedFirstWhen FileSourceSplit fourthSplit = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2020-12-31/hr=23/ahaha4a5dg"), 0, 1024); ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); - ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder.addPathParser(new HourDatePathParser()).build(Collections.singleton(secondSplit)); + ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder + .addPathParser(new HourDatePathParser()) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(Collections.singleton(secondSplit)); List remainingSplitsToAdd = Arrays.asList(fourthSplit, firstSplit, thirdSplit); splitAssigner.addSplits(remainingSplitsToAdd); @@ -143,7 +164,10 @@ public void shouldReturnRemainingSplitsWhichAreNotAssignedYetInAscendingOrderOfF ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); - ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder.addPathParser(new HourDatePathParser()).build(inputSplits); + ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder + .addPathParser(new HourDatePathParser()) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(inputSplits); splitAssigner.getNext(null); List remainingSplits = (List) splitAssigner.remainingSplits(); @@ -160,7 +184,10 @@ public void shouldCallPathParserToParseFilePathURL() throws ParseException { ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); - splitAssignerBuilder.addPathParser(hourDatePathParser).build(Collections.singleton(split)); + splitAssignerBuilder + .addPathParser(hourDatePathParser) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(Collections.singleton(split)); verify(hourDatePathParser, times(1)).instantFromFilePath(split.path()); } @@ -180,6 +207,7 @@ public void shouldReturnFileSplitsFallingInGivenTimeRangeWhenFilePathURLHasBothD ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder .addPathParser(new HourDatePathParser()) .addTimeRanges(timeRangePool) + .addStatsDReporterSupplier(statsDReporterSupplierMock) .build(inputSplits); assertEquals(firstSplit, splitAssigner.getNext(null).get()); @@ -204,6 +232,7 @@ public void shouldReturnFileSplitsFallingInMultipleTimeRangesWhenFilePathURLHasB ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder .addPathParser(new HourDatePathParser()) .addTimeRanges(timeRangePool) + .addStatsDReporterSupplier(statsDReporterSupplierMock) .build(inputSplits); assertEquals(firstSplit, splitAssigner.getNext(null).get()); @@ -224,8 +253,11 @@ public void shouldThrowExceptionIfPathParserNotProvided() { TimeRangePool timeRangePool = new TimeRangePool(); timeRangePool.add(new TimeRange(Instant.parse("2019-10-12T00:00:00Z"), Instant.parse("2019-10-12T04:00:00Z"))); timeRangePool.add(new TimeRange(Instant.parse("2020-11-29T00:00:00Z"), Instant.parse("2020-11-30T10:00:00Z"))); - PathParserNotProvidedException pathParserNotProvidedException = assertThrows(PathParserNotProvidedException.class, () -> splitAssignerBuilder.addTimeRanges(timeRangePool).build(inputSplits)); + PathParserNotProvidedException pathParserNotProvidedException = assertThrows(PathParserNotProvidedException.class, + () -> splitAssignerBuilder + .addTimeRanges(timeRangePool) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(inputSplits)); assertEquals("Path parser is null", pathParserNotProvidedException.getMessage()); } - } From 36f237c5b656cbe2342109a214807388c26b9011 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 12:55:02 +0530 Subject: [PATCH 27/47] feat: rename interface method [https://github.com/odpf/dagger/issues/108] --- .../core/metrics/reporters/statsd/DaggerStatsDReporter.java | 2 +- .../reporters/statsd/SerializedStatsDReporterSupplier.java | 2 +- .../core/metrics/reporters/statsd/StatsDErrorReporter.java | 2 +- .../metrics/reporters/statsd/manager/DaggerCounterManager.java | 2 +- .../metrics/reporters/statsd/manager/DaggerGaugeManager.java | 2 +- .../reporters/statsd/manager/DaggerHistogramManager.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java index 5dae82ad0..b369671d9 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java @@ -30,7 +30,7 @@ private String[] generateGlobalTags() { } @Override - public StatsDReporter getStatsDReporter() { + public StatsDReporter buildStatsDReporter() { if (statsDReporter == null) { DaggerMetricsConfig daggerMetricsConfig = new DaggerMetricsConfig(flinkConfiguration); String[] globalTags = generateGlobalTags(); diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java index 84ebbd23d..42ca8f93c 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java @@ -6,5 +6,5 @@ @FunctionalInterface public interface SerializedStatsDReporterSupplier extends Serializable { - StatsDReporter getStatsDReporter(); + StatsDReporter buildStatsDReporter(); } diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java index 682f8dbad..6bfa205f1 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java @@ -15,7 +15,7 @@ public class StatsDErrorReporter implements ErrorReporter { private final StatsDReporter statsDReporter; public StatsDErrorReporter(SerializedStatsDReporterSupplier statsDReporterSupplier) { - this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); + this.statsDReporter = statsDReporterSupplier.buildStatsDReporter(); } @Override diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java index 3c7579598..1110beeae 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManager.java @@ -13,7 +13,7 @@ public class DaggerCounterManager implements MeasurementManager, Counter { private String[] formattedTags; public DaggerCounterManager(SerializedStatsDReporterSupplier statsDReporterSupplier) { - this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); + this.statsDReporter = statsDReporterSupplier.buildStatsDReporter(); } @Override diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java index 004e5180f..10c7c6771 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManager.java @@ -13,7 +13,7 @@ public class DaggerGaugeManager implements MeasurementManager, Gauge { private String[] formattedTags; public DaggerGaugeManager(SerializedStatsDReporterSupplier statsDReporterSupplier) { - this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); + this.statsDReporter = statsDReporterSupplier.buildStatsDReporter(); } @Override diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java index 0beebca72..93b8b5065 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManager.java @@ -13,7 +13,7 @@ public class DaggerHistogramManager implements MeasurementManager, Histogram { private String[] formattedTags; public DaggerHistogramManager(SerializedStatsDReporterSupplier statsDReporterSupplier) { - this.statsDReporter = statsDReporterSupplier.getStatsDReporter(); + this.statsDReporter = statsDReporterSupplier.buildStatsDReporter(); } @Override From e8f960214dd806ab28a9b39b386054a024189fa1 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 13:10:02 +0530 Subject: [PATCH 28/47] feat: add tests for DaggerStatsDReporter [https://github.com/odpf/dagger/issues/108] --- .../statsd/DaggerStatsDReporterTest.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterTest.java diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterTest.java new file mode 100644 index 000000000..4d51d5d84 --- /dev/null +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterTest.java @@ -0,0 +1,55 @@ +package io.odpf.dagger.core.metrics.reporters.statsd; + +import org.apache.flink.configuration.Configuration; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +public class DaggerStatsDReporterTest { + + @Mock + private Configuration flinkConfiguration; + + @Mock + private io.odpf.dagger.common.configuration.Configuration daggerConfiguration; + + @Before + public void setup() { + initMocks(this); + } + + @Test + public void shouldBeAbleToBuildAndMaintainSingletonCopyOfStatsDReporter() { + when(daggerConfiguration.getString(anyString(), anyString())).thenReturn("some-tag"); + + DaggerStatsDReporter daggerStatsDReporter = DaggerStatsDReporter.Provider.provide(flinkConfiguration, daggerConfiguration); + + assertEquals(daggerStatsDReporter.buildStatsDReporter(), daggerStatsDReporter.buildStatsDReporter()); + + } + + @Test + public void shouldBuildStatsDReporterWithGlobalTags() { + when(daggerConfiguration.getString(anyString(), anyString())).thenReturn("some-tag"); + + DaggerStatsDReporter.Provider + .provide(flinkConfiguration, daggerConfiguration) + .buildStatsDReporter(); + + ArgumentCaptor jobIdCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor defaultJobIdCaptor = ArgumentCaptor.forClass(String.class); + + verify(daggerConfiguration, times(1)).getString(jobIdCaptor.capture(), defaultJobIdCaptor.capture()); + + assertEquals("FLINK_JOB_ID", jobIdCaptor.getValue()); + assertEquals("SQL Flink job", defaultJobIdCaptor.getValue()); + } +} From 365c533d58c6c61c28313943341adbe77802539b Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 13:27:46 +0530 Subject: [PATCH 29/47] feat: add tests for DaggerMetricsConfig [https://github.com/odpf/dagger/issues/108] --- .../statsd/DaggerMetricsConfigTest.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfigTest.java diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfigTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfigTest.java new file mode 100644 index 000000000..bbc5c5414 --- /dev/null +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerMetricsConfigTest.java @@ -0,0 +1,65 @@ +package io.odpf.dagger.core.metrics.reporters.statsd; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.Configuration; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + + +public class DaggerMetricsConfigTest { + @Mock + private Configuration flinkConfiguration; + + @Before + public void setup() { + initMocks(this); + } + + @Test + public void shouldSetHostNameAsDefinedInFlinkConfiguration() { + ConfigOption hostConfigOption = ConfigOptions + .key("metrics.reporter.stsd.host") + .stringType() + .defaultValue("localhost"); + when(flinkConfiguration.getString(hostConfigOption)).thenReturn("my-host"); + + DaggerMetricsConfig daggerMetricsConfig = new DaggerMetricsConfig(flinkConfiguration); + assertEquals("my-host", daggerMetricsConfig.getMetricStatsDHost()); + } + + @Test + public void shouldUseDefaultHostNameWhenNotDefinedInFlinkConfiguration() { + DaggerMetricsConfig daggerMetricsConfig = new DaggerMetricsConfig(new Configuration()); + assertEquals("localhost", daggerMetricsConfig.getMetricStatsDHost()); + } + + @Test + public void shouldSetPortNumberAsDefinedInFlinkConfiguration() { + ConfigOption portConfigOption = ConfigOptions + .key("metrics.reporter.stsd.port") + .intType() + .defaultValue(8125); + when(flinkConfiguration.getInteger(portConfigOption)).thenReturn(9010); + + DaggerMetricsConfig daggerMetricsConfig = new DaggerMetricsConfig(flinkConfiguration); + assertEquals(9010, daggerMetricsConfig.getMetricStatsDPort().intValue()); + } + + @Test + public void shouldUseDefaultPortWhenNotDefinedInFlinkConfiguration() { + DaggerMetricsConfig daggerMetricsConfig = new DaggerMetricsConfig(new Configuration()); + assertEquals(8125, daggerMetricsConfig.getMetricStatsDPort().intValue()); + } + + @Test + public void shouldReturnEmptyStringAsTags() { + DaggerMetricsConfig daggerMetricsConfig = new DaggerMetricsConfig(new Configuration()); + assertEquals("", daggerMetricsConfig.getMetricStatsDTags()); + } +} From cea2b4ba01c291b3897ff929ff9b0770ff0c97bf Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 15:08:42 +0530 Subject: [PATCH 30/47] feat: add tests for DaggerCounterManager [https://github.com/odpf/dagger/issues/108] --- .../manager/DaggerCounterManagerTest.java | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java new file mode 100644 index 000000000..453333793 --- /dev/null +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java @@ -0,0 +1,81 @@ +package io.odpf.dagger.core.metrics.reporters.statsd.manager; + +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; +import io.odpf.depot.metrics.StatsDReporter; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +import static io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects.READER_CLOSED; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +public class DaggerCounterManagerTest { + @Mock + private StatsDReporter statsDReporter; + + @Before + public void setup() { + initMocks(this); + } + + private SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; + + @Test + public void shouldIncrementCounterMeasurement() { + DaggerCounterManager daggerCounterManager = new DaggerCounterManager(statsDReporterSupplier); + + daggerCounterManager.increment(READER_CLOSED); + + verify(statsDReporter, times(1)).captureCount(READER_CLOSED.getValue(), 1L, (String[]) null); + } + + @Test + public void shouldIncrementCounterMeasurementWithDelta() { + DaggerCounterManager daggerCounterManager = new DaggerCounterManager(statsDReporterSupplier); + + daggerCounterManager.increment(READER_CLOSED, 5L); + + verify(statsDReporter, times(1)).captureCount(READER_CLOSED.getValue(), 5L, (String[]) null); + } + + @Test + public void shouldIncrementCounterMeasurementWithRegisteredTags() { + DaggerCounterManager daggerCounterManager = new DaggerCounterManager(statsDReporterSupplier); + daggerCounterManager.register(new StatsDTag[]{new StatsDTag("tag1", "value1"), new StatsDTag("tag2", "value2")}); + + daggerCounterManager.increment(READER_CLOSED); + + verify(statsDReporter, times(1)).captureCount(READER_CLOSED.getValue(), 1L, "tag1=value1", "tag2=value2"); + } + + @Test + public void shouldDecrementCounterMeasurement() { + DaggerCounterManager daggerCounterManager = new DaggerCounterManager(statsDReporterSupplier); + + daggerCounterManager.decrement(READER_CLOSED); + + verify(statsDReporter, times(1)).captureCount(READER_CLOSED.getValue(), -1L, (String[]) null); + } + + @Test + public void shouldDecrementCounterMeasurementWithDelta() { + DaggerCounterManager daggerCounterManager = new DaggerCounterManager(statsDReporterSupplier); + + daggerCounterManager.decrement(READER_CLOSED, -5L); + + verify(statsDReporter, times(1)).captureCount(READER_CLOSED.getValue(), -5L, (String[]) null); + } + + @Test + public void shouldDecrementCounterMeasurementWithRegisteredTags() { + DaggerCounterManager daggerCounterManager = new DaggerCounterManager(statsDReporterSupplier); + daggerCounterManager.register(new StatsDTag[]{new StatsDTag("tag1", "value1"), new StatsDTag("tag2", "value2")}); + + daggerCounterManager.decrement(READER_CLOSED); + + verify(statsDReporter, times(1)).captureCount(READER_CLOSED.getValue(), -1L, "tag1=value1", "tag2=value2"); + } +} \ No newline at end of file From 9fd90047460437d2be5c0cb17ac1d0fb4e02cbf8 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 16:24:44 +0530 Subject: [PATCH 31/47] feat: add a close method to DaggerStatsDReporter for cleanup [https://github.com/odpf/dagger/issues/108] --- .../statsd/DaggerStatsDReporter.java | 8 +++++++ .../statsd/DaggerStatsDReporterTest.java | 24 +++++++++---------- .../manager/DaggerCounterManagerTest.java | 2 +- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java index b369671d9..733cacdf3 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporter.java @@ -6,6 +6,7 @@ import io.odpf.depot.metrics.StatsDReporterBuilder; import org.apache.flink.configuration.Configuration; +import java.io.IOException; import java.util.Arrays; import static io.odpf.dagger.core.utils.Constants.FLINK_JOB_ID_DEFAULT; @@ -43,6 +44,13 @@ public StatsDReporter buildStatsDReporter() { return statsDReporter; } + protected static void close() throws IOException { + if (statsDReporter != null) { + statsDReporter.close(); + statsDReporter = null; + } + } + public static class Provider { public static DaggerStatsDReporter provide(Configuration flinkConfiguration, io.odpf.dagger.common.configuration.Configuration daggerConfiguration) { return new DaggerStatsDReporter(flinkConfiguration, daggerConfiguration); diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterTest.java index 4d51d5d84..ac8f2b3f0 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/DaggerStatsDReporterTest.java @@ -6,6 +6,8 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; +import java.io.IOException; + import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.times; @@ -22,24 +24,14 @@ public class DaggerStatsDReporterTest { private io.odpf.dagger.common.configuration.Configuration daggerConfiguration; @Before - public void setup() { + public void setup() throws IOException { initMocks(this); - } - - @Test - public void shouldBeAbleToBuildAndMaintainSingletonCopyOfStatsDReporter() { + DaggerStatsDReporter.close(); when(daggerConfiguration.getString(anyString(), anyString())).thenReturn("some-tag"); - - DaggerStatsDReporter daggerStatsDReporter = DaggerStatsDReporter.Provider.provide(flinkConfiguration, daggerConfiguration); - - assertEquals(daggerStatsDReporter.buildStatsDReporter(), daggerStatsDReporter.buildStatsDReporter()); - } @Test public void shouldBuildStatsDReporterWithGlobalTags() { - when(daggerConfiguration.getString(anyString(), anyString())).thenReturn("some-tag"); - DaggerStatsDReporter.Provider .provide(flinkConfiguration, daggerConfiguration) .buildStatsDReporter(); @@ -52,4 +44,12 @@ public void shouldBuildStatsDReporterWithGlobalTags() { assertEquals("FLINK_JOB_ID", jobIdCaptor.getValue()); assertEquals("SQL Flink job", defaultJobIdCaptor.getValue()); } + + @Test + public void shouldBeAbleToBuildAndMaintainSingletonCopyOfStatsDReporter() { + DaggerStatsDReporter daggerStatsDReporter = DaggerStatsDReporter.Provider + .provide(flinkConfiguration, daggerConfiguration); + + assertEquals(daggerStatsDReporter.buildStatsDReporter(), daggerStatsDReporter.buildStatsDReporter()); + } } diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java index 453333793..1733d047d 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java @@ -78,4 +78,4 @@ public void shouldDecrementCounterMeasurementWithRegisteredTags() { verify(statsDReporter, times(1)).captureCount(READER_CLOSED.getValue(), -1L, "tag1=value1", "tag2=value2"); } -} \ No newline at end of file +} From e13ed529214a73af9952436c0cb733de3946b329 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 16:25:52 +0530 Subject: [PATCH 32/47] feat: delete meter measurement [https://github.com/odpf/dagger/issues/108] --- .../metrics/reporters/statsd/measurement/Meter.java | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Meter.java diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Meter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Meter.java deleted file mode 100644 index 14bc4830b..000000000 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/measurement/Meter.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.odpf.dagger.core.metrics.reporters.statsd.measurement; - -import io.odpf.dagger.common.metrics.aspects.Aspects; - -import java.io.Serializable; - -public interface Meter extends Serializable { - void markEvent(Aspects aspect); - - void markEvent(Aspects aspect, long num); -} From cf1d76da5dfc77900c073e973611ac7a5eb6e505 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 16:32:03 +0530 Subject: [PATCH 33/47] feat: add tests for DaggerGaugeManager [https://github.com/odpf/dagger/issues/108] --- .../manager/DaggerGaugeManagerTest.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManagerTest.java diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManagerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManagerTest.java new file mode 100644 index 000000000..af1d6ab95 --- /dev/null +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManagerTest.java @@ -0,0 +1,44 @@ +package io.odpf.dagger.core.metrics.reporters.statsd.manager; + +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; +import io.odpf.depot.metrics.StatsDReporter; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.TOTAL_SPLITS_DISCOVERED; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +public class DaggerGaugeManagerTest { + @Mock + private StatsDReporter statsDReporter; + + @Before + public void setup() { + initMocks(this); + } + + private SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; + + @Test + public void shouldMarkGaugeValue() { + DaggerGaugeManager daggerGaugeManager = new DaggerGaugeManager(statsDReporterSupplier); + + daggerGaugeManager.markValue(TOTAL_SPLITS_DISCOVERED, 1000); + + verify(statsDReporter, times(1)).gauge(TOTAL_SPLITS_DISCOVERED.getValue(), 1000, (String[]) null); + } + + @Test + public void shouldMarkGaugeValueWithRegisteredTags() { + DaggerGaugeManager daggerGaugeManager = new DaggerGaugeManager(statsDReporterSupplier); + daggerGaugeManager.register(new StatsDTag[]{new StatsDTag("tag1", "value1"), new StatsDTag("tag2", "value2")}); + + daggerGaugeManager.markValue(TOTAL_SPLITS_DISCOVERED, 1000); + + verify(statsDReporter, times(1)).gauge(TOTAL_SPLITS_DISCOVERED.getValue(), 1000, "tag1=value1", "tag2=value2"); + } +} From 1529f0d15292063d5c3bb9240edcd614b5ecf454 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 16:51:24 +0530 Subject: [PATCH 34/47] feat: add tests for DaggerHistogramManager [https://github.com/odpf/dagger/issues/108] --- .../manager/DaggerHistogramManagerTest.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManagerTest.java diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManagerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManagerTest.java new file mode 100644 index 000000000..49a0aca81 --- /dev/null +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManagerTest.java @@ -0,0 +1,44 @@ +package io.odpf.dagger.core.metrics.reporters.statsd.manager; + +import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; +import io.odpf.dagger.core.metrics.reporters.statsd.tags.StatsDTag; +import io.odpf.depot.metrics.StatsDReporter; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +import static io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects.READER_ROW_READ_TIME; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +public class DaggerHistogramManagerTest { + @Mock + private StatsDReporter statsDReporter; + + @Before + public void setup() { + initMocks(this); + } + + private SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; + + @Test + public void shouldRecordHistogramValue() { + DaggerHistogramManager daggerHistogramManager = new DaggerHistogramManager(statsDReporterSupplier); + + daggerHistogramManager.recordValue(READER_ROW_READ_TIME, 5L); + + verify(statsDReporter, times(1)).captureHistogram(READER_ROW_READ_TIME.getValue(), 5L, (String[]) null); + } + + @Test + public void shouldRecordHistogramValueWithRegisteredTags() { + DaggerHistogramManager daggerHistogramManager = new DaggerHistogramManager(statsDReporterSupplier); + daggerHistogramManager.register(new StatsDTag[]{new StatsDTag("tag1", "value1"), new StatsDTag("tag2", "value2")}); + + daggerHistogramManager.recordValue(READER_ROW_READ_TIME, 6L); + + verify(statsDReporter, times(1)).captureHistogram(READER_ROW_READ_TIME.getValue(), 6L, "tag1=value1", "tag2=value2"); + } +} From fa9f19d20e92f5eab724a8fc7f22c4d2e8985e9e Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 16:52:17 +0530 Subject: [PATCH 35/47] feat: make instance variable final in tests [https://github.com/odpf/dagger/issues/108] --- .../reporters/statsd/manager/DaggerCounterManagerTest.java | 2 +- .../reporters/statsd/manager/DaggerGaugeManagerTest.java | 2 +- .../reporters/statsd/manager/DaggerHistogramManagerTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java index 1733d047d..49e84bf81 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerCounterManagerTest.java @@ -21,7 +21,7 @@ public void setup() { initMocks(this); } - private SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; + private final SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; @Test public void shouldIncrementCounterMeasurement() { diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManagerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManagerTest.java index af1d6ab95..9658cf87f 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManagerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerGaugeManagerTest.java @@ -21,7 +21,7 @@ public void setup() { initMocks(this); } - private SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; + private final SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; @Test public void shouldMarkGaugeValue() { diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManagerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManagerTest.java index 49a0aca81..571c6be6e 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManagerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/manager/DaggerHistogramManagerTest.java @@ -21,7 +21,7 @@ public void setup() { initMocks(this); } - private SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; + private final SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; @Test public void shouldRecordHistogramValue() { From c32948815cd9e502e4e9507c846313fc8c4a3135 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 17:07:59 +0530 Subject: [PATCH 36/47] feat: change exception message [https://github.com/odpf/dagger/issues/108] --- .../core/metrics/reporters/statsd/StatsDErrorReporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java index 6bfa205f1..060a9a077 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java @@ -32,6 +32,6 @@ public void reportNonFatalException(Exception exception) { @Override public Counter addExceptionToCounter(Exception exception, MetricGroup metricGroup, String metricGroupKey) { - throw new UnsupportedOperationException("This operation is not supported on StatsDReporter"); + throw new UnsupportedOperationException("This operation is not supported on StatsDErrorReporter"); } } From 482c833133ef8160ac60510bb953112a7a9abdae Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 17:08:24 +0530 Subject: [PATCH 37/47] feat: add tests for StatsDErrorReporter [https://github.com/odpf/dagger/issues/108] --- .../statsd/StatsDErrorReporterTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporterTest.java diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporterTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporterTest.java new file mode 100644 index 000000000..e8def56b7 --- /dev/null +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporterTest.java @@ -0,0 +1,56 @@ +package io.odpf.dagger.core.metrics.reporters.statsd; + +import io.odpf.depot.metrics.StatsDReporter; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; + +public class StatsDErrorReporterTest { + @Mock + private StatsDReporter statsDReporter; + + @Before + public void setup() { + initMocks(this); + } + + private final SerializedStatsDReporterSupplier statsDReporterSupplier = () -> statsDReporter; + + @Test + public void shouldReportFatalExceptionWithExceptionClassNameAsTagValue() { + StatsDErrorReporter statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); + + statsDErrorReporter.reportFatalException(new NullPointerException("This is a fatal exception")); + + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, + "fatal_exception_type=" + NullPointerException.class.getName()); + } + + @Test + public void shouldReportNonFatalExceptionWithExceptionClassNameAsTagValue() { + StatsDErrorReporter statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); + + statsDErrorReporter.reportNonFatalException(new Exception("This is a non-fatal exception")); + + verify(statsDReporter, times(1)) + .captureCount("non.fatal.exception", 1L, + "non_fatal_exception_type=" + Exception.class.getName()); + } + + @Test + public void shouldThrowUnsupportedExceptionForAddExceptionToCounter() { + StatsDErrorReporter statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); + + UnsupportedOperationException ex = assertThrows(UnsupportedOperationException.class, + () -> statsDErrorReporter.addExceptionToCounter(null, null, null)); + + assertEquals("This operation is not supported on StatsDErrorReporter", ex.getMessage()); + } +} From 1308032f8ec053ee2dc99b70f6fa477111ec66db Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 18:41:33 +0530 Subject: [PATCH 38/47] feat: add tests for ChronologyOrderedSplitAssigner [https://github.com/odpf/dagger/issues/108] --- .../ChronologyOrderedSplitAssignerTest.java | 80 ++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssignerTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssignerTest.java index 09fa72d4f..34ab3bade 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssignerTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/splitassigner/ChronologyOrderedSplitAssignerTest.java @@ -10,7 +10,9 @@ import org.apache.flink.connector.file.src.FileSourceSplit; import org.apache.flink.core.fs.Path; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mock; import org.mockito.Mockito; import java.text.ParseException; @@ -20,14 +22,25 @@ import java.util.List; import java.util.Optional; +import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.SPLITS_AWAITING_ASSIGNMENT; +import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.TOTAL_SPLITS_DISCOVERED; +import static io.odpf.dagger.core.metrics.aspects.ChronologyOrderedSplitAssignerAspects.TOTAL_SPLITS_RECORDED; import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.MockitoAnnotations.initMocks; public class ChronologyOrderedSplitAssignerTest { - private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); + @Mock + private StatsDReporter statsDReporter; + + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> statsDReporter; + + @Before + public void setup() { + initMocks(this); + } @Test public void shouldReturnFileSplitsHavingOldestDateFilePathsFirstWhenFilePathURLHasOnlyDate() { @@ -260,4 +273,67 @@ public void shouldThrowExceptionIfPathParserNotProvided() { .build(inputSplits)); assertEquals("Path parser is null", pathParserNotProvidedException.getMessage()); } + + @Test + public void shouldRaiseMetricsForDiscoveredSplitsAndRecordedSplitsWithTagsWhenInitialized() { + FileSourceSplit firstSplit = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2019-10-12/hr=00/hd6a7gad"), 0, 1024); + FileSourceSplit secondSplit = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2019-10-12/hr=08/sa6advgad7"), 0, 1024); + FileSourceSplit thirdSplit = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2020-11-30/hr=09/aga6adgad"), 0, 1024); + FileSourceSplit fourthSplit = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2020-12-31/hr=23/ahaha4a5dg"), 0, 1024); + List inputSplits = Arrays.asList(secondSplit, fourthSplit, firstSplit, thirdSplit); + + TimeRangePool timeRangePool = new TimeRangePool(); + timeRangePool.add(new TimeRange(Instant.parse("2019-10-12T00:00:00Z"), Instant.parse("2020-11-30T10:00:00Z"))); + + ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); + + splitAssignerBuilder + .addPathParser(new HourDatePathParser()) + .addTimeRanges(timeRangePool) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(inputSplits); + + verify(statsDReporter, times(1)).gauge(TOTAL_SPLITS_DISCOVERED.getValue(), 4, "component=split_assigner"); + verify(statsDReporter, times(1)).gauge(TOTAL_SPLITS_RECORDED.getValue(), 3, "component=split_assigner"); + } + + @Test + public void shouldRaiseMetricsWithTagsAfterAssigningSplits() { + FileSourceSplit firstSplit = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2019-10-12/asdghsdhasd"), 0, 1024); + FileSourceSplit secondSplit = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2020-02-29/ga6agad6ad"), 0, 1024); + List inputSplits = Arrays.asList(firstSplit, secondSplit); + + ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); + ChronologyOrderedSplitAssigner splitAssigner = splitAssignerBuilder.addPathParser(new HourDatePathParser()) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(inputSplits); + splitAssigner.getNext(null); + + verify(statsDReporter, times(1)).gauge(SPLITS_AWAITING_ASSIGNMENT.getValue(), 1, "component=split_assigner"); + } + + @Test + public void shouldRaiseErrorMetricsWhenFilePathValidationFailed() { + FileSourceSplit split = new FileSourceSplit("1", new Path("gs://my-bucket/bid-log/dt=2019-130-12/shs6s5sdg"), 0, 1024); + ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); + assertThrows(IllegalArgumentException.class, () -> splitAssignerBuilder + .addPathParser(new HourDatePathParser()) + .addStatsDReporterSupplier(statsDReporterSupplierMock) + .build(Collections.singleton(split))); + + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + IllegalArgumentException.class.getName()); + } + + @Test + public void shouldThrowExceptionIfStatsDSupplierNotProvided() { + ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder splitAssignerBuilder = new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder(); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> splitAssignerBuilder + .addPathParser(new HourDatePathParser()) + .build(Collections.emptyList())); + + assertEquals("SerializedStatsDReporterSupplier is required but is set as null", exception.getMessage()); + } } From 5085eb4babc17043fb191a978d5ac094ddca94dc Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 18:57:05 +0530 Subject: [PATCH 39/47] feat: add tests for StatsDTag - refactor flow for null tag value [https://github.com/odpf/dagger/issues/108] --- .../reporters/statsd/tags/StatsDTag.java | 11 +--- .../reporters/statsd/tags/StatsDTagTest.java | 51 +++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTagTest.java diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java index bec416416..d1c40b22e 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java @@ -8,16 +8,9 @@ public class StatsDTag { private static final String NIL_TAG_VALUE = "NIL_TAG_VALUE"; public StatsDTag(String key, String value) { - validate(key, value); + Preconditions.checkArgument(key != null && !key.isEmpty(), "Tag key cannot be null or empty"); this.tagKey = key; - this.tagValue = value; - } - - private void validate(String key, String value) { - Preconditions.checkArgument(key != null && !key.isEmpty(), - "Tag key cannot be null or empty"); - Preconditions.checkArgument(value != null && !value.isEmpty(), - "Tag value cannot be null or empty. Refer other constructors for creating a tag with only key"); + this.tagValue = (value != null && !value.isEmpty()) ? value : NIL_TAG_VALUE; } public StatsDTag(String tagName) { diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTagTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTagTest.java new file mode 100644 index 000000000..1db97bd8d --- /dev/null +++ b/dagger-core/src/test/java/io/odpf/dagger/core/metrics/reporters/statsd/tags/StatsDTagTest.java @@ -0,0 +1,51 @@ +package io.odpf.dagger.core.metrics.reporters.statsd.tags; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class StatsDTagTest { + + @Test + public void shouldReturnFormattedTagWithKeyValueSeparatedByDelimiter() { + StatsDTag statsDTag = new StatsDTag("tag-key", "tag-value"); + + assertEquals("tag-key=tag-value", statsDTag.getFormattedTag()); + } + + @Test + public void shouldReturnFormattedTagWithOnlyTagKey() { + StatsDTag statsDTag = new StatsDTag("my-tag"); + + assertEquals("my-tag", statsDTag.getFormattedTag()); + } + + @Test + public void shouldReturnFormattedTagWithOnlyTagKeyWhenTagValueIsNull() { + StatsDTag statsDTag = new StatsDTag("my-tag", null); + + assertEquals("my-tag", statsDTag.getFormattedTag()); + } + + @Test + public void shouldReturnFormattedTagWithOnlyTagKeyWhenTagValueIsEmpty() { + StatsDTag statsDTag = new StatsDTag("my-tag", ""); + + assertEquals("my-tag", statsDTag.getFormattedTag()); + } + + @Test + public void shouldThrowIllegalArgumentExceptionWhenTagKeyIsNull() { + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> new StatsDTag(null, "tag-value")); + + assertEquals("Tag key cannot be null or empty", ex.getMessage()); + } + + @Test + public void shouldThrowIllegalArgumentExceptionWhenTagKeyIsEmpty() { + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> new StatsDTag("", "tag-value")); + + assertEquals("Tag key cannot be null or empty", ex.getMessage()); + } +} From 230c3f83caf38c021e8015b9f99fbfdcbe418370 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 19:01:16 +0530 Subject: [PATCH 40/47] feat: add error reporter tests for DaggerSourceFactory [https://github.com/odpf/dagger/issues/108] --- .../dagger/core/source/DaggerSourceFactoryTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java index 6ba8432c7..e95d1e3d7 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/DaggerSourceFactoryTest.java @@ -21,7 +21,8 @@ import org.mockito.Mockito; import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -32,7 +33,10 @@ public class DaggerSourceFactoryTest { @Mock private Configuration configuration; - private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); + @Mock + private StatsDReporter statsDReporter; + + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> statsDReporter; @Before public void setUp() throws Exception { @@ -67,10 +71,12 @@ public void shouldReturnParquetDaggerSourceWhenConfigured() { } @Test - public void shouldThrowRuntimeExceptionIfNoDaggerSourceCouldBeCreatedAsPerConfigs() { + public void shouldThrowRuntimeExceptionAndReportErrorIfNoDaggerSourceCouldBeCreatedAsPerConfigs() { SimpleGroupDeserializer deserializer = Mockito.mock(SimpleGroupDeserializer.class); when(streamConfig.getSourceDetails()).thenReturn(new SourceDetails[]{new SourceDetails(SourceName.PARQUET_SOURCE, SourceType.UNBOUNDED)}); assertThrows(InvalidDaggerSourceException.class, () -> DaggerSourceFactory.create(streamConfig, configuration, deserializer, statsDReporterSupplierMock)); + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + InvalidDaggerSourceException.class.getName()); } } From 5e76f05325dee93c8cac12303ad5e5aacd9ac225 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 19:18:48 +0530 Subject: [PATCH 41/47] feat: add error reporter tests for ParquetDaggerSource [https://github.com/odpf/dagger/issues/108] --- .../core/source/parquet/ParquetDaggerSourceTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java index b731dd96f..22a0f5f33 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetDaggerSourceTest.java @@ -48,7 +48,10 @@ public class ParquetDaggerSourceTest { @Mock private StreamExecutionEnvironment streamExecutionEnvironment; - private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); + @Mock + private StatsDReporter statsDReporter; + + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> statsDReporter; private FileSource fileSource; @@ -127,12 +130,14 @@ public void shouldUseStreamConfigurationToBuildTheFileSource() { } @Test - public void shouldThrowRuntimeExceptionIfReadOrderStrategyIsNotSupported() { + public void shouldThrowRuntimeExceptionAndReportErrorIfReadOrderStrategyIsNotSupported() { when(streamConfig.getParquetFilesReadOrderStrategy()).thenReturn(EARLIEST_INDEX_FIRST); when(streamConfig.getParquetFilePaths()).thenReturn(new String[]{"gs://sshsh", "gs://shadd"}); ParquetDaggerSource daggerSource = new ParquetDaggerSource(streamConfig, configuration, daggerDeserializer, statsDReporterSupplierMock); assertThrows(DaggerConfigurationException.class, () -> daggerSource.register(streamExecutionEnvironment, strategy)); + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + DaggerConfigurationException.class.getName()); } } From 5704371c9ecda4d2d387669bf1be09a8ecc22ea3 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 19:29:56 +0530 Subject: [PATCH 42/47] feat: add error reporter tests for ParquetFileRecordFormat [https://github.com/odpf/dagger/issues/108] --- .../parquet/ParquetFileRecordFormatTest.java | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java index 19577921a..59a5ad7f1 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormatTest.java @@ -17,7 +17,8 @@ import java.util.function.Supplier; import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.MockitoAnnotations.initMocks; public class ParquetFileRecordFormatTest { @@ -31,9 +32,12 @@ public class ParquetFileRecordFormatTest { @Mock private Configuration configuration; + @Mock + private StatsDReporter statsDReporter; + private final ReaderProvider readerProviderMock = (filePath) -> parquetReader; private final Supplier> typeInformationProviderMock = () -> typeInformation; - private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> statsDReporter; @Before public void setup() { @@ -56,7 +60,7 @@ public void shouldBuildAFileRecordFormatAsPerConfiguredParameters() { } @Test - public void shouldThrowIllegalArgumentExceptionWhenReaderProviderIsNotConfigured() { + public void shouldThrowIllegalArgumentExceptionAndReportErrorWhenReaderProviderIsNotConfigured() { ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, @@ -66,10 +70,12 @@ public void shouldThrowIllegalArgumentExceptionWhenReaderProviderIsNotConfigured .build()); assertEquals("ReaderProvider is required but is set as null", ex.getMessage()); + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + IllegalArgumentException.class.getName()); } @Test - public void shouldThrowIllegalArgumentExceptionWhenTypeInformationProviderIsNotConfigured() { + public void shouldThrowIllegalArgumentExceptionAndReportErrorWhenTypeInformationProviderIsNotConfigured() { ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder @@ -78,6 +84,20 @@ public void shouldThrowIllegalArgumentExceptionWhenTypeInformationProviderIsNotC .build()); assertEquals("TypeInformationProvider is required but is set as null", ex.getMessage()); + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + IllegalArgumentException.class.getName()); + } + + @Test + public void shouldThrowIllegalArgumentExceptionWhenStatsDReporterSupplierIsNotConfigured() { + ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> builder + .setParquetFileReaderProvider(readerProviderMock) + .setTypeInformationProvider(typeInformationProviderMock) + .build()); + + assertEquals("SerializedStatsDReporterSupplier is required but is set as null", ex.getMessage()); } @Test @@ -92,7 +112,7 @@ public void shouldReturnFalseWhenIsSplittableIsCalled() { } @Test - public void shouldThrowUnsupportedOperationExceptionWhenRestoreReaderIsCalled() { + public void shouldThrowUnsupportedOperationExceptionAndReportErrorWhenRestoreReaderIsCalled() { ParquetFileRecordFormat.Builder builder = ParquetFileRecordFormat.Builder.getInstance(); ParquetFileRecordFormat parquetFileRecordFormat = builder.setTypeInformationProvider(typeInformationProviderMock) .setParquetFileReaderProvider(readerProviderMock) @@ -103,5 +123,7 @@ public void shouldThrowUnsupportedOperationExceptionWhenRestoreReaderIsCalled() () -> parquetFileRecordFormat.restoreReader(configuration, new Path("gs://some-path"), 12, 0, 1024)); assertEquals("Error: ParquetReader do not have offsets and hence cannot be restored via this method.", ex.getMessage()); + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + UnsupportedOperationException.class.getName()); } } From 6acf9dba13aaf6b9d8fe6c12ceac2c56ca6876f0 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 19:41:17 +0530 Subject: [PATCH 43/47] feat: add error reporter tests for ParquetFileSource [https://github.com/odpf/dagger/issues/108] --- .../source/parquet/ParquetFileSourceTest.java | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileSourceTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileSourceTest.java index ed2bd5ed0..2cda4ba4c 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileSourceTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/ParquetFileSourceTest.java @@ -17,7 +17,8 @@ import static java.util.Collections.emptyList; import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.MockitoAnnotations.initMocks; public class ParquetFileSourceTest { @@ -27,7 +28,10 @@ public class ParquetFileSourceTest { @Mock private FileRecordFormat fileRecordFormat; - private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); + @Mock + private StatsDReporter statsDReporter; + + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> statsDReporter; @Before public void setup() { @@ -58,7 +62,7 @@ public void shouldBuildParquetFileSourceAsPerArguments() { } @Test - public void shouldThrowExceptionIfSourceTypeConfiguredAsUnbounded() { + public void shouldThrowExceptionAndReportErrorIfSourceTypeConfiguredAsUnbounded() { Builder builder = Builder.getInstance(); Path[] filePaths = new Path[]{new Path("gs://aadadc"), new Path("gs://sjsjhd")}; @@ -72,10 +76,12 @@ public void shouldThrowExceptionIfSourceTypeConfiguredAsUnbounded() { .build()); assertEquals("Running Parquet FileSource in UNBOUNDED mode is not supported yet", ex.getMessage()); + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + IllegalArgumentException.class.getName()); } @Test - public void shouldThrowExceptionIfFileRecordFormatIsNotSet() { + public void shouldThrowExceptionAndReportErrorIfFileRecordFormatIsNotSet() { Builder builder = Builder.getInstance(); Path[] filePaths = new Path[]{new Path("gs://aadadc"), new Path("gs://sjsjhd")}; @@ -88,10 +94,12 @@ public void shouldThrowExceptionIfFileRecordFormatIsNotSet() { .build()); assertEquals("FileRecordFormat is required but is set as null", ex.getMessage()); + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + IllegalArgumentException.class.getName()); } @Test - public void shouldThrowExceptionIfNoFilePathsSet() { + public void shouldThrowExceptionAndReportErrorIfNoFilePathsSet() { Builder builder = Builder.getInstance(); IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, @@ -103,6 +111,24 @@ public void shouldThrowExceptionIfNoFilePathsSet() { .build()); assertEquals("At least one file path is required but none are provided", ex.getMessage()); + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + IllegalArgumentException.class.getName()); + } + + @Test + public void shouldThrowExceptionIfStatsDReporterSupplierIsNotSet() { + Builder builder = Builder.getInstance(); + Path[] filePaths = new Path[]{new Path("gs://aadadc"), new Path("gs://sjsjhd")}; + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> builder.setConfiguration(configuration) + .setFileRecordFormat(fileRecordFormat) + .setSourceType(SourceType.BOUNDED) + .setFileSplitAssigner(new ChronologyOrderedSplitAssigner.ChronologyOrderedSplitAssignerBuilder()::build) + .setFilePaths(filePaths) + .build()); + + assertEquals("SerializedStatsDReporterSupplier is required but is set as null", ex.getMessage()); } @Test From 9e98b97ef4f267cc02288d6feb029e7ff6d97cd6 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 20:59:22 +0530 Subject: [PATCH 44/47] feat: add error reporter tests for ParquetReader [https://github.com/odpf/dagger/issues/108] --- .../parquet/reader/ParquetReaderTest.java | 78 ++++++++++++++++++- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java index 3145cc192..66413cab2 100644 --- a/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java +++ b/dagger-core/src/test/java/io/odpf/dagger/core/source/parquet/reader/ParquetReaderTest.java @@ -1,6 +1,7 @@ package io.odpf.dagger.core.source.parquet.reader; +import io.odpf.dagger.common.exceptions.serde.DaggerDeserializationException; import io.odpf.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; import io.odpf.dagger.common.serde.parquet.deserialization.SimpleGroupDeserializer; import io.odpf.dagger.core.exception.ParquetFileSourceReaderInitializationException; @@ -17,17 +18,22 @@ import org.junit.rules.TemporaryFolder; import org.mockito.ArgumentCaptor; import org.mockito.Mock; +import org.mockito.Mockito; import java.io.File; import java.io.IOException; import java.util.List; +import static io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects.READER_CLOSED; +import static io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects.READER_CREATED; +import static io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects.READER_ROWS_EMITTED; +import static io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects.READER_ROW_DESERIALIZATION_TIME; +import static io.odpf.dagger.core.metrics.aspects.ParquetReaderAspects.READER_ROW_READ_TIME; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; import static org.apache.parquet.schema.Types.*; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -40,13 +46,75 @@ public class ParquetReaderTest { @Rule public TemporaryFolder tempFolder = TemporaryFolder.builder().assureDeletion().build(); - private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> mock(StatsDReporter.class); + @Mock + private StatsDReporter statsDReporter; + + private final SerializedStatsDReporterSupplier statsDReporterSupplierMock = () -> statsDReporter; @Before public void setup() { initMocks(this); } + @Test + public void shouldRaiseMetricsWhenInitialized() { + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); + ClassLoader classLoader = getClass().getClassLoader(); + String filePath = classLoader.getResource("test_file.parquet").getPath(); + + provider.getReader(filePath); + + verify(statsDReporter, Mockito.times(1)).captureCount(READER_CREATED.getValue(), 1L, "component=parquet_reader"); + } + + @Test + public void shouldRaiseMetricsWhenReadingFileAndDeserializingToRows() throws IOException { + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); + ClassLoader classLoader = getClass().getClassLoader(); + String filePath = classLoader.getResource("test_file.parquet").getPath(); + + ParquetReader reader = provider.getReader(filePath); + reader.read(); + + verify(statsDReporter, Mockito.times(1)).captureCount(READER_ROWS_EMITTED.getValue(), 1L, "component=parquet_reader"); + + ArgumentCaptor measurementNameCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor executionTimeCaptor = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor tagCaptor = ArgumentCaptor.forClass(String.class); + + verify(statsDReporter, Mockito.times(2)).captureHistogram(measurementNameCaptor.capture(), executionTimeCaptor.capture(), tagCaptor.capture()); + + assertEquals(READER_ROW_READ_TIME.getValue(), measurementNameCaptor.getAllValues().get(0)); + assertEquals(READER_ROW_DESERIALIZATION_TIME.getValue(), measurementNameCaptor.getAllValues().get(1)); + assertEquals("component=parquet_reader", tagCaptor.getAllValues().get(0)); + assertEquals("component=parquet_reader", tagCaptor.getAllValues().get(1)); + } + + @Test + public void shouldRaiseMetricsWhenClosingTheReader() throws IOException { + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); + ClassLoader classLoader = getClass().getClassLoader(); + String filePath = classLoader.getResource("test_file.parquet").getPath(); + + provider.getReader(filePath).close(); + + verify(statsDReporter, Mockito.times(1)).captureCount(READER_CLOSED.getValue(), 1L, "component=parquet_reader"); + } + + @Test + public void shouldReportErrorAndRethrowWhenDaggerDeserializerThrowsException() { + when(deserializer.deserialize(any(SimpleGroup.class))).thenThrow(DaggerDeserializationException.class); + + ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); + ClassLoader classLoader = getClass().getClassLoader(); + String filePath = classLoader.getResource("test_file.parquet").getPath(); + + assertThrows(DaggerDeserializationException.class, () -> provider.getReader(filePath).read()); + + verify(statsDReporter, Mockito.times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + DaggerDeserializationException.class.getName()); + } + @Test public void shouldCreateReadersConfiguredWithTheSameDeserializerButForDifferentFilePaths() throws IOException { when(deserializer.deserialize(any(SimpleGroup.class))).thenReturn(Row.of("same", "deserializer")); @@ -140,11 +208,13 @@ public void shouldThrowIOExceptionIfReadIsCalledAfterCallingClose() throws IOExc } @Test - public void shouldThrowParquetFileSourceReaderInitializationExceptionIfCannotConstructReaderForTheFile() throws IOException { + public void shouldThrowParquetFileSourceReaderInitializationExceptionAndReportErrorIfCannotConstructReaderForTheFile() throws IOException { final File tempFile = tempFolder.newFile("test_file.parquet"); ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); assertThrows(ParquetFileSourceReaderInitializationException.class, () -> provider.getReader(tempFile.getPath())); + verify(statsDReporter, times(1)) + .captureCount("fatal.exception", 1L, "fatal_exception_type=" + ParquetFileSourceReaderInitializationException.class.getName()); } @Test @@ -172,7 +242,7 @@ public void shouldUpdateCheckPointedPositionWithNoOffsetAndCountOfTotalRecordsRe } @Test - public void shouldNotUpdateCheckpointedPositionWhenNoMoreRecordsToRead() throws IOException { + public void shouldNotUpdateCheckPointedPositionWhenNoMoreRecordsToRead() throws IOException { ParquetReader.ParquetReaderProvider provider = new ParquetReader.ParquetReaderProvider(deserializer, statsDReporterSupplierMock); ClassLoader classLoader = getClass().getClassLoader(); ParquetReader reader = provider.getReader(classLoader.getResource("test_file.parquet").getPath()); From b8540221bb0a36b0526a47bf802e499067953c4e Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Fri, 8 Jul 2022 21:00:06 +0530 Subject: [PATCH 45/47] feat: delete unused measurements [https://github.com/odpf/dagger/issues/108] --- .../dagger/core/metrics/aspects/ParquetReaderAspects.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java index b44a32fa1..9a755e843 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/aspects/ParquetReaderAspects.java @@ -8,9 +8,7 @@ public enum ParquetReaderAspects implements Aspects { READER_CLOSED("reader_closed", AspectType.Counter), READER_ROWS_EMITTED("reader_rows_emitted", AspectType.Counter), READER_ROW_DESERIALIZATION_TIME("reader_row_deserialization_time", AspectType.Histogram), - READER_ROW_READ_TIME("reader_row_read_time", AspectType.Histogram), - READER_CREATION_EXCEPTION("reader_creation_exception", AspectType.Counter), - READER_DESERIALIZATION_EXCEPTION("reader_deserialization_exception", AspectType.Counter); + READER_ROW_READ_TIME("reader_row_read_time", AspectType.Histogram); private final String value; private final AspectType aspectType; From a4cd68713a004982ee1b2473ff6195d7500d674c Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Mon, 11 Jul 2022 12:56:08 +0530 Subject: [PATCH 46/47] feat: use lambda to provide error reporter in ParquetFileRecordFormat - FileRecordFormat object and all it's fields need to be serializable in order to construct the Flink job graph. Even though StatsDErrorReporter is made serializable, it contains StatsDErrorReporter, which in turn contains more fields which may not be serializable.Hence, in order to mitigate job graph creation failures, we wrap the error reporter inside a serializable lambda. This is a common idiom to make un-serializable fields serializable in Java 8: https://stackoverflow.com/a/22808112 [https://github.com/odpf/dagger/issues/108] --- .../statsd/SerializedStatsDReporterSupplier.java | 5 +++++ .../reporters/statsd/StatsDErrorReporter.java | 4 +++- .../core/source/parquet/ParquetFileRecordFormat.java | 12 +++++++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java index 42ca8f93c..7991bf293 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java @@ -4,6 +4,11 @@ import java.io.Serializable; +/* Flink requires that all objects which are needed to prepare the Job Graph should be serializable along with their +properties/fields. StatsDReporter and its fields are not serializable. Hence, in order to mitigate job graph creation +failure, we create a serializable interface around the reporter as below. This is a common idiom to make un-serializable +fields serializable in Java 8: https://stackoverflow.com/a/22808112 */ + @FunctionalInterface public interface SerializedStatsDReporterSupplier extends Serializable { StatsDReporter buildStatsDReporter(); diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java index 060a9a077..53856695d 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/metrics/reporters/statsd/StatsDErrorReporter.java @@ -6,10 +6,12 @@ import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.MetricGroup; +import java.io.Serializable; + import static io.odpf.dagger.core.utils.Constants.FATAL_EXCEPTION_METRIC_GROUP_KEY; import static io.odpf.dagger.core.utils.Constants.NONFATAL_EXCEPTION_METRIC_GROUP_KEY; -public class StatsDErrorReporter implements ErrorReporter { +public class StatsDErrorReporter implements ErrorReporter, Serializable { private static final String FATAL_EXCEPTION_TAG_KEY = "fatal_exception_type"; private static final String NON_FATAL_EXCEPTION_TAG_KEY = "non_fatal_exception_type"; private final StatsDReporter statsDReporter; diff --git a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java index 77a5fd84a..e93b9eb81 100644 --- a/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java +++ b/dagger-core/src/main/java/io/odpf/dagger/core/source/parquet/ParquetFileRecordFormat.java @@ -11,17 +11,23 @@ import org.apache.flink.core.fs.Path; import org.apache.flink.types.Row; +import java.io.Serializable; import java.util.function.Supplier; public class ParquetFileRecordFormat implements FileRecordFormat { + /* FileRecordFormat object and all it's fields need to be serializable in order to construct the Flink job graph. Even though + StatsDErrorReporter is serializable, it contains StatsDErrorReporter, which in turn contains more fields which may not be + serializable.Hence, in order to mitigate job graph creation failures, we wrap the error reporter inside a serializable lambda. + This is a common idiom to make un-serializable fields serializable in Java 8: https://stackoverflow.com/a/22808112 */ + private final ReaderProvider parquetFileReaderProvider; private final Supplier> typeInformationProvider; - private final StatsDErrorReporter statsDErrorReporter; + private final Supplier statsDErrorReporterSupplier; private ParquetFileRecordFormat(ReaderProvider parquetFileReaderProvider, Supplier> typeInformationProvider, SerializedStatsDReporterSupplier statsDReporterSupplier) { this.parquetFileReaderProvider = parquetFileReaderProvider; this.typeInformationProvider = typeInformationProvider; - this.statsDErrorReporter = new StatsDErrorReporter(statsDReporterSupplier); + this.statsDErrorReporterSupplier = (Supplier & Serializable) () -> new StatsDErrorReporter(statsDReporterSupplier); } @Override @@ -33,7 +39,7 @@ public Reader createReader(Configuration config, Path filePath, long splitO public Reader restoreReader(Configuration config, Path filePath, long restoredOffset, long splitOffset, long splitLength) { UnsupportedOperationException ex = new UnsupportedOperationException("Error: ParquetReader do not have offsets and hence cannot be restored " + "via this method."); - statsDErrorReporter.reportFatalException(ex); + statsDErrorReporterSupplier.get().reportFatalException(ex); throw ex; } From 2aafa622ba586d33894d2e9533108327dd9ee9a0 Mon Sep 17 00:00:00 2001 From: Meghajit Mazumdar Date: Mon, 11 Jul 2022 14:14:46 +0530 Subject: [PATCH 47/47] doc: add documentation for metrics [https://github.com/odpf/dagger/issues/108] --- docs/docs/reference/metrics.md | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/docs/reference/metrics.md b/docs/docs/reference/metrics.md index 25a84e6db..498ce0c28 100644 --- a/docs/docs/reference/metrics.md +++ b/docs/docs/reference/metrics.md @@ -16,6 +16,7 @@ Service-level Indicators \(SLIs\) are the measurements used to calculate the per - [Processors](metrics.md#processors) - [Longbow](metrics.md#longbow) - [Checkpointing](metrics.md#checkpointing) +- [Parquet Source](metrics.md#parquet-source) ## Overview @@ -435,6 +436,49 @@ This shows the details about checkpointing of the dagger job. Checkpointing is F - The stream where the input topic is read from +## Parquet Source + +This shows the metrics related to Parquet Source in Dagger. The metrics included are: + +### Row Deserialization Time Percentiles + +- This shows the percentile distribution of the time duration required to deserialize a single parquet record from a file +into a row. + +### Rows Emitted by Source Per Second + +- This shows, as the title suggests, the number of rows emitted by each source-reader aka the task manager per second. This +acts as a measure of the throughput of the parquet source. + +### Parquet Record Read Time Percentiles + +- This shows the percentile distribution of the time duration required to read a single parquet record from a file. + +### Total File Splits Discovered + +- This shows the net total of all files discovered by Dagger after doing a recursive search on all the GCS URLs as specified by the +config `SOURCE_PARQUET_FILE_PATHS`. + +### Total File Splits To Be Processed + +- This shows the net total of all files which will be processed by this dagger job. It is basically the number of file +paths as shown by the metric [`Total File Splits Discovered`](metrics.md#total-file-splits-discovered) minus the files which were skipped after applying the time +range filter as specified by config [`SOURCE_PARQUET_FILE_DATE_RANGE`](configuration.md#source_parquet_file_date_range). + +### File Splits Remaining For Processing + +- This shows the number of files which are pending to be processed by Dagger. It is a live metric and as the job progresses, +this value will slowly come down as the files get processed by Dagger and ultimately become 0 when the job completes. + +### Readers Created Per Minute + +- This shows the number of Parquet File readers which are created by Dagger per minute to process the files. One +reader is created for each new file. Readers are not re-used. + +### Readers Closed Per Minute + +- This shows the number of Parquet File readers which are closed and cleaned up by Dagger per minute. + ## Overview Some of the most important metrics related to firehose that gives you an overview of the current state of it.