Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ public void shouldPopulateFieldFromESOnSuccessResponse() throws Exception {
" \"socket_timeout\": \"5000\",\n" +
" \"retry_timeout\": \"5000\",\n" +
" \"capacity\": \"30\",\n" +
" \"type\": \"com.gojek.esb.fraud.EnrichedBookingLogMessage\", \n" +
" \"type\": \"com.gojek..fraud.EnrichedBookingLogMessage\", \n" +
" \"output_mapping\": {\n" +
" \"customer_profile\": {\n" +
" \"path\": \"$._source\"\n" +
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import com.gojek.esb.consumer.TestGrpcRequest;
import com.gojek.esb.consumer.TestGrpcResponse;
import com.gojek.esb.consumer.TestServerGrpc;
import io.odpf.dagger.consumer.TestGrpcRequest;
import io.odpf.dagger.consumer.TestGrpcResponse;
import io.odpf.dagger.consumer.TestServerGrpc;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
import org.apache.flink.streaming.api.datastream.DataStream;
Expand Down Expand Up @@ -50,7 +50,7 @@ public class GrpcExternalPostProcessorIntegrationTest {

@Before
public void setUp() {
String streams = "[{\"TOPIC_NAMES\":\"SG_GO_CAR-booking-log\",\"TABLE_NAME\":\"booking\",\"PROTO_CLASS_NAME\":\"com.gojek.esb.booking.BookingLogMessage\",\"EVENT_TIMESTAMP_FIELD_INDEX\":\"41\",\"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\":\"10.200.216.49:6668,10.200.219.198:6668,10.200.216.58:6668,10.200.216.54:6668,10.200.216.56:6668,10.200.216.63:6668\",\"KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE\":\"\",\"KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET\":\"latest\",\"KAFKA_CONSUMER_CONFIG_GROUP_ID\":\"test-config\",\"STREAM_NAME\":\"p-godata-id-mainstream\"}]";
String streams = "[{\"TOPIC_NAMES\":\"SG_GO_CAR-booking-log\",\"TABLE_NAME\":\"booking\",\"PROTO_CLASS_NAME\":\"io.odpf.dagger.booking.BookingLogMessage\",\"EVENT_TIMESTAMP_FIELD_INDEX\":\"41\",\"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\":\"10.200.216.49:6668,10.200.219.198:6668,10.200.216.58:6668,10.200.216.54:6668,10.200.216.56:6668,10.200.216.63:6668\",\"KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE\":\"\",\"KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET\":\"latest\",\"KAFKA_CONSUMER_CONFIG_GROUP_ID\":\"test-config\",\"STREAM_NAME\":\"p-godata-id-mainstream\"}]";
configuration.setString(POST_PROCESSOR_ENABLED_KEY, "true");
configuration.setString(INPUT_STREAMS, streams);
configuration.setString(STENCIL_ENABLE_KEY, "false");
Expand All @@ -74,9 +74,9 @@ public void shouldPopulateFieldFromGrpcOnSuccess() throws Exception {
" \"fail_on_errors\": false,\n" +
" \"retain_response_type\": true,\n" +
" \"grpc_stencil_url\": \"http://localhost:8000/messages.desc\",\n" +
" \"grpc_request_proto_schema\": \"com.gojek.esb.consumer.TestGrpcRequest\",\n" +
" \"grpc_response_proto_schema\": \"com.gojek.esb.consumer.TestGrpcResponse\",\n" +
" \"grpc_method_url\": \"com.gojek.esb.consumer.TestServer/TestRpcMethod\",\n" +
" \"grpc_request_proto_schema\": \"io.odpf.dagger.consumer.TestGrpcRequest\",\n" +
" \"grpc_response_proto_schema\": \"io.odpf.dagger.consumer.TestGrpcResponse\",\n" +
" \"grpc_method_url\": \"io.odpf.dagger.consumer.TestServer/TestRpcMethod\",\n" +
" \"capacity\": \"30\",\n" +
" \"headers\": {\n" +
" \"content-type\": \"application/json\" \n" +
Expand Down Expand Up @@ -141,9 +141,9 @@ public void shouldPopulateFieldFromGrpcOnSuccessWithExternalAndInternalSource()
" \"fail_on_errors\": false,\n" +
" \"retain_response_type\": true,\n" +
" \"grpc_stencil_url\": \"http://localhost:8000/messages.desc\",\n" +
" \"grpc_request_proto_schema\": \"com.gojek.esb.consumer.TestGrpcRequest\",\n" +
" \"grpc_response_proto_schema\": \"com.gojek.esb.consumer.TestGrpcResponse\",\n" +
" \"grpc_method_url\": \"com.gojek.esb.consumer.TestServer/TestRpcMethod\",\n" +
" \"grpc_request_proto_schema\": \"io.odpf.dagger.consumer.TestGrpcRequest\",\n" +
" \"grpc_response_proto_schema\": \"io.odpf.dagger.consumer.TestGrpcResponse\",\n" +
" \"grpc_method_url\": \"io.odpf.dagger.consumer.TestServer/TestRpcMethod\",\n" +
" \"capacity\": \"30\",\n" +
" \"output_mapping\": {\n" +
" \"field3\": {\n" +
Expand Down Expand Up @@ -223,9 +223,9 @@ public void shouldPopulateFieldFromGrpcOnSuccessWithAllThreeSourcesIncludingTran
" \"fail_on_errors\": false,\n" +
" \"retain_response_type\": true,\n" +
" \"grpc_stencil_url\": \"http://localhost:8000/messages.desc\",\n" +
" \"grpc_request_proto_schema\": \"com.gojek.esb.consumer.TestGrpcRequest\",\n" +
" \"grpc_response_proto_schema\": \"com.gojek.esb.consumer.TestGrpcResponse\",\n" +
" \"grpc_method_url\": \"com.gojek.esb.consumer.TestServer/TestRpcMethod\",\n" +
" \"grpc_request_proto_schema\": \"io.odpf.dagger.consumer.TestGrpcRequest\",\n" +
" \"grpc_response_proto_schema\": \"io.odpf.dagger.consumer.TestGrpcResponse\",\n" +
" \"grpc_method_url\": \"io.odpf.dagger.consumer.TestServer/TestRpcMethod\",\n" +
" \"capacity\": \"30\",\n" +
" \"output_mapping\": {\n" +
" \"field3\": {\n" +
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.odpf.dagger.config.system;

import com.gojek.daggers.config.EnvironmentConfigurationProvider;
import io.odpf.dagger.config.EnvironmentConfigurationProvider;
import org.apache.flink.configuration.Configuration;
import org.junit.Test;
import org.junit.runner.RunWith;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package io.odpf.dagger.config.system;

import com.gojek.daggers.config.EnvironmentConfigurationProvider;
import com.gojek.daggers.config.FileConfigurationProvider;
import io.odpf.dagger.config.FileConfigurationProvider;
import org.apache.flink.configuration.Configuration;
import org.junit.Test;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,9 @@ public void shouldReturnMultiURLStencilClient() throws NoSuchFieldException, Ill
when(configuration.getString(STENCIL_CONFIG_TTL_IN_MINUTES_KEY, STENCIL_CONFIG_TTL_IN_MINUTES_DEFAULT)).thenReturn("30");
when(configuration.getString(STENCIL_CONFIG_TIMEOUT_MS_KEY, STENCIL_CONFIG_TIMEOUT_MS_DEFAULT)).thenReturn(STENCIL_CONFIG_TIMEOUT_MS_DEFAULT);
when(configuration.getBoolean(STENCIL_ENABLE_KEY, STENCIL_ENABLE_DEFAULT)).thenReturn(true);
when(configuration.getString(STENCIL_URL_KEY, STENCIL_URL_DEFAULT)).thenReturn("http://artifactory-gojek.golabs.io/artifactory/proto-descriptors/esb-log-entities/latest," +
"http://artifactory-gojek.golabs.io/artifactory/proto-descriptors/goid-events/latest," +
"http://artifactory-gojek.golabs.io/artifactory/proto-descriptors/growth-log-entities/release");
when(configuration.getString(STENCIL_URL_KEY, STENCIL_URL_DEFAULT)).thenReturn("http://localhost/latest," +
"http://localhost/events/latest," +
"http://localhost/entities/release");
StencilClientOrchestrator stencilClientOrchestrator = new StencilClientOrchestrator(configuration);
stencilClient = stencilClientOrchestrator.getStencilClient();

Expand All @@ -71,13 +71,13 @@ public void shouldEnrichStencilClient() throws NoSuchFieldException, IllegalAcce
when(configuration.getString(STENCIL_CONFIG_TTL_IN_MINUTES_KEY, STENCIL_CONFIG_TTL_IN_MINUTES_DEFAULT)).thenReturn("30");
when(configuration.getString(STENCIL_CONFIG_TIMEOUT_MS_KEY, STENCIL_CONFIG_TIMEOUT_MS_DEFAULT)).thenReturn(STENCIL_CONFIG_TIMEOUT_MS_DEFAULT);
when(configuration.getBoolean(STENCIL_ENABLE_KEY, STENCIL_ENABLE_DEFAULT)).thenReturn(true);
when(configuration.getString(STENCIL_URL_KEY, STENCIL_URL_DEFAULT)).thenReturn("http://artifactory-gojek.golabs.io/artifactory/proto-descriptors/esb-log-entities/latest,");
when(configuration.getString(STENCIL_URL_KEY, STENCIL_URL_DEFAULT)).thenReturn("http://localhost/latest,");
StencilClientOrchestrator stencilClientOrchestrator = new StencilClientOrchestrator(configuration);
StencilClient oldStencilClient = stencilClientOrchestrator.getStencilClient();


List<String> enrichmentStencilURLs = Collections
.singletonList("http://stencil.golabs.io/artifactory/proto-descriptors/feast-proto/latest");
.singletonList("http://localhost/latest");

Assert.assertSame(oldStencilClient, stencilClientOrchestrator.getStencilClient());

Expand All @@ -99,7 +99,7 @@ public void shouldNotEnrichIfNoNewAdditionalURLsAdded() throws NoSuchFieldExcept
when(configuration.getString(STENCIL_CONFIG_TTL_IN_MINUTES_KEY, STENCIL_CONFIG_TTL_IN_MINUTES_DEFAULT)).thenReturn("30");
when(configuration.getString(STENCIL_CONFIG_TIMEOUT_MS_KEY, STENCIL_CONFIG_TIMEOUT_MS_DEFAULT)).thenReturn(STENCIL_CONFIG_TIMEOUT_MS_DEFAULT);
when(configuration.getBoolean(STENCIL_ENABLE_KEY, STENCIL_ENABLE_DEFAULT)).thenReturn(true);
when(configuration.getString(STENCIL_URL_KEY, STENCIL_URL_DEFAULT)).thenReturn("http://artifactory-gojek.golabs.io/artifactory/proto-descriptors/esb-log-entities/latest,");
when(configuration.getString(STENCIL_URL_KEY, STENCIL_URL_DEFAULT)).thenReturn("http://localhost/latest,");
StencilClientOrchestrator stencilClientOrchestrator = new StencilClientOrchestrator(configuration);
StencilClient oldStencilClient = stencilClientOrchestrator.getStencilClient();

Expand Down Expand Up @@ -130,7 +130,7 @@ public void shouldReturnClassLoadStencilClientWhenStencilDisabledAndEnrichmentSt
StencilClientOrchestrator stencilClientOrchestrator = new StencilClientOrchestrator(configuration);

List<String> enrichmentStencilURLs = Collections
.singletonList("http://stencil.golabs.io/artifactory/proto-descriptors/feast-proto/latest");
.singletonList("http://localhost/latest");

StencilClient stencilClient = stencilClientOrchestrator.enrichStencilClient(enrichmentStencilURLs);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ public class StreamManagerTest {
+ " \"EVENT_TIMESTAMP_FIELD_INDEX\": \"4\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE\": \"false\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET\": \"latest\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"p-esb-kafka-mirror-b-01:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"localhost:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_GROUP_ID\": \"flink-sql-flud-gp0330\",\n"
+ " \"PROTO_CLASS_NAME\": \"com.gojek.esb.booking.BookingLogMessage\",\n"
+ " \"PROTO_CLASS_NAME\": \"io.odpf.dagger.consumer.TestBookingLogMessage\",\n"
+ " \"TABLE_NAME\": \"data_stream\",\n"
+ " \"TOPIC_NAMES\": \"GO_RIDE-booking-log\"\n"
+ " }\n"
Expand Down
28 changes: 14 additions & 14 deletions dagger-core/src/test/java/io/odpf/dagger/core/StreamsTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ public void shouldTakeAJSONArrayWithSingleObject() {
+ " \"EVENT_TIMESTAMP_FIELD_INDEX\": \"4\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE\": \"false\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET\": \"latest\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"p-esb-kafka-mirror-b-01:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"localhost:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_GROUP_ID\": \"flink-sql-flud-gp0330\",\n"
+ " \"PROTO_CLASS_NAME\": \"com.gojek.esb.booking.BookingLogMessage\",\n"
+ " \"PROTO_CLASS_NAME\": \"io.odpf.dagger.consumer.TestBookingLogMessage\",\n"
+ " \"TABLE_NAME\": \"data_stream\",\n"
+ " \"TOPIC_NAMES\": \"GO_RIDE-booking-log\"\n"
+ " }\n"
Expand All @@ -66,9 +66,9 @@ public void shouldAddTopicsStreamsAndProtosToMetrics() {
+ " \"EVENT_TIMESTAMP_FIELD_INDEX\": \"4\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE\": \"false\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET\": \"latest\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"p-esb-kafka-mirror-b-01:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"localhost:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_GROUP_ID\": \"flink-sql-flud-gp0330\",\n"
+ " \"PROTO_CLASS_NAME\": \"com.gojek.esb.booking.BookingLogMessage\",\n"
+ " \"PROTO_CLASS_NAME\": \"io.odpf.dagger.consumer.TestBookingLogMessage\",\n"
+ " \"TABLE_NAME\": \"data_stream\",\n"
+ " \"TOPIC_NAMES\": \"GO_RIDE-booking-log\"\n"
+ " }\n"
Expand All @@ -77,7 +77,7 @@ public void shouldAddTopicsStreamsAndProtosToMetrics() {
ArrayList<String> topicNames = new ArrayList<>();
topicNames.add("GO_RIDE-booking-log");
ArrayList<String> protoName = new ArrayList<>();
protoName.add("com.gojek.esb.booking.BookingLogMessage");
protoName.add("io.odpf.dagger.consumer.TestBookingLogMessage");
ArrayList<String> streamName = new ArrayList<>();
streamName.add("");
HashMap<String, List<String>> metrics = new HashMap<>();
Expand All @@ -103,9 +103,9 @@ public void shouldAddTopicsStreamsAndProtosToMetricsInCaseOfJoins() {
+ " \"EVENT_TIMESTAMP_FIELD_INDEX\": \"4\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE\": \"false\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET\": \"latest\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"p-esb-kafka-mirror-b-01:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"localhost:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_GROUP_ID\": \"flink-sql-flud-gp0330\",\n"
+ " \"PROTO_CLASS_NAME\": \"com.gojek.esb.booking.BookingLogMessage\",\n"
+ " \"PROTO_CLASS_NAME\": \"io.odpf.dagger.consumer.TestBookingLogMessage\",\n"
+ " \"TABLE_NAME\": \"data_stream\",\n"
+ " \"STREAM_NAME\": \"mainstream\",\n"
+ " \"TOPIC_NAMES\": \"GO_RIDE-booking-log\"\n"
Expand All @@ -114,9 +114,9 @@ public void shouldAddTopicsStreamsAndProtosToMetricsInCaseOfJoins() {
+ " \"EVENT_TIMESTAMP_FIELD_INDEX\": \"1\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE\": \"false\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET\": \"latest\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"p-esb-kafka-mirror-b-01:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"localhost:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_GROUP_ID\": \"flink-sql-flud-gp0330\",\n"
+ " \"PROTO_CLASS_NAME\": \"com.gojek.esb.aggregate.surge.SurgeFactorLogMessage\",\n"
+ " \"PROTO_CLASS_NAME\": \"io.odpf.dagger.consumer.TestBookingLogKey\",\n"
+ " \"TABLE_NAME\": \"data_stream_1\",\n"
+ " \"STREAM_NAME\": \"locstream\",\n"
+ " \"TOPIC_NAMES\": \"surge-s2idcluster-log\"\n"
Expand All @@ -127,8 +127,8 @@ public void shouldAddTopicsStreamsAndProtosToMetricsInCaseOfJoins() {
topicNames.add("GO_RIDE-booking-log");
topicNames.add("surge-s2idcluster-log");
ArrayList<String> protoName = new ArrayList<>();
protoName.add("com.gojek.esb.booking.BookingLogMessage");
protoName.add("com.gojek.esb.aggregate.surge.SurgeFactorLogMessage");
protoName.add("io.odpf.dagger.consumer.TestBookingLogMessage");
protoName.add("io.odpf.dagger.consumer.TestBookingLogKey");
ArrayList<String> streamName = new ArrayList<>();
streamName.add("mainstream");
streamName.add("locstream");
Expand All @@ -155,16 +155,16 @@ public void shouldReturnProtoClassName() {
+ " \"EVENT_TIMESTAMP_FIELD_INDEX\": \"4\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE\": \"false\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET\": \"latest\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"p-esb-kafka-mirror-b-01:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS\": \"localhost:6667\",\n"
+ " \"KAFKA_CONSUMER_CONFIG_GROUP_ID\": \"flink-sql-flud-gp0330\",\n"
+ " \"PROTO_CLASS_NAME\": \"com.gojek.esb.booking.BookingLogMessage\",\n"
+ " \"PROTO_CLASS_NAME\": \"io.odpf.dagger.consumer.TestBookingLogMessage\",\n"
+ " \"TABLE_NAME\": \"data_stream\",\n"
+ " \"TOPIC_NAMES\": \"GO_RIDE-booking-log\"\n"
+ " }\n"
+ "]";

LinkedHashMap<String, String> protoClassForTable = new LinkedHashMap<>();
protoClassForTable.put("data_stream", "com.gojek.esb.booking.BookingLogMessage");
protoClassForTable.put("data_stream", "io.odpf.dagger.consumer.TestBookingLogMessage");

configuration = new Configuration();
configuration.setString("STREAMS", configString);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ public class PreProcessorFactoryTest {
" \"table_transformers\": [{\n" +
" \"table_name\": \"booking\",\n" +
" \"transformers\": [{\n" +
" \"transformation_class\": \"com.gojek.daggers.PreProcessorClass\"\n" +
" \"transformation_class\": \"PreProcessorClass\"\n" +
" }, {\n" +
" \"transformation_class\": \"com.gojek.daggers.PreProcessorClass\",\n" +
" \"transformation_class\": \"PreProcessorClass\",\n" +
" \"transformation_arguments\": {\n" +
" \"key\": \"value\"\n" +
" }\n" +
Expand All @@ -46,7 +46,7 @@ public class PreProcessorFactoryTest {
" {\n" +
" \"table_name\": \"another_booking\",\n" +
" \"transformers\": [{\n" +
" \"transformation_class\": \"com.gojek.daggers.PreProcessorClass\"\n" +
" \"transformation_class\": \"PreProcessorClass\"\n" +
" }]\n" +
" }\n" +
" ]\n" +
Expand Down Expand Up @@ -74,7 +74,7 @@ public void shouldParseConfig() {
PreProcessorConfig preProcessorConfig = PreProcessorFactory.parseConfig(configuration);
Assert.assertEquals(2, preProcessorConfig.getTableTransformers().size());
Assert.assertEquals(2, preProcessorConfig.getTableTransformers().get(0).getTransformers().size());
Assert.assertEquals("com.gojek.daggers.PreProcessorClass", preProcessorConfig.getTableTransformers().get(0).getTransformers().get(0).getTransformationClass());
Assert.assertEquals("PreProcessorClass", preProcessorConfig.getTableTransformers().get(0).getTransformers().get(0).getTransformationClass());
}

@Test
Expand Down
Loading