From a60cfb43319a51e84ca18e319a1080ed666ef558 Mon Sep 17 00:00:00 2001
From: stym06 <stym06>
Date: Mon, 27 Feb 2023 12:01:41 +0530
Subject: [PATCH] make protoc plugin compatiable for M1 chip & fix some tc
---
build.gradle | 14 +++++++++++--
.../serializer/MessageToJsonTest.java | 20 +++++++++----------
.../firehose/sink/grpc/GrpcClientTest.java | 4 ++--
.../sink/grpc/GrpcSinkFactoryTest.java | 2 +-
4 files changed, 25 insertions(+), 15 deletions(-)
diff --git a/build.gradle b/build.gradle
index 09f4d0d81..85672d038 100644
--- a/build.gradle
+++ b/build.gradle
@@ -93,6 +93,7 @@ dependencies {
implementation(group: 'org.apache.hadoop', name: 'hadoop-common', version: '3.1.0') {
exclude group: "log4j", module: "log4j"
exclude group: "org.slf4j", module: "slf4j-log4j12"
+ exclude group: "org.xerial.snappy", module: "snappy-java"
}
implementation group: 'com.gojek.parquet', name: 'parquet-protobuf', version: '1.11.9'
implementation 'com.gojek.parquet:parquet-hadoop:1.11.9'
@@ -103,6 +104,7 @@ dependencies {
implementation 'org.apache.logging.log4j:log4j-core:2.17.1'
implementation group: 'io.odpf', name: 'depot', version: '0.3.7'
implementation group: 'com.networknt', name: 'json-schema-validator', version: '1.0.59' exclude group: 'org.slf4j'
+ implementation 'org.xerial.snappy:snappy-java:1.1.8.4'
testImplementation group: 'junit', name: 'junit', version: '4.11'
testImplementation 'org.hamcrest:hamcrest-all:1.3'
@@ -117,11 +119,19 @@ dependencies {
protobuf {
generatedFilesBaseDir = "$projectDir/src/generated"
protoc {
- artifact = "com.google.protobuf:protoc:3.1.0"
+ if (osdetector.os == "osx") {
+ artifact = 'com.google.protobuf:protoc:3.1.0:osx-x86_64'
+ } else {
+ artifact = "com.google.protobuf:protoc:3.1.0"
+ }
}
plugins {
grpc {
- artifact = "io.grpc:protoc-gen-grpc-java:1.0.3"
+ if (osdetector.os == "osx") {
+ artifact = 'io.grpc:protoc-gen-grpc-java:1.0.3:osx-x86_64'
+ } else {
+ artifact = "io.grpc:protoc-gen-grpc-java:1.0.3"
+ }
}
}
generateProtoTasks {
diff --git a/src/test/java/io/odpf/firehose/serializer/MessageToJsonTest.java b/src/test/java/io/odpf/firehose/serializer/MessageToJsonTest.java
index c14689ca6..388e8cb65 100644
--- a/src/test/java/io/odpf/firehose/serializer/MessageToJsonTest.java
+++ b/src/test/java/io/odpf/firehose/serializer/MessageToJsonTest.java
@@ -34,11 +34,11 @@ public void shouldProperlySerializeEsbMessage() throws DeserializerException {
Base64.getDecoder().decode(logMessage.getBytes()), "sample-topic", 0, 100);
String actualOutput = messageToJson.serialize(message);
assertEquals(actualOutput, "{\"logMessage\":\"{\\\"uniqueDrivers\\\":\\\"3\\\","
- + "\\\"windowStartTime\\\":\\\"Mar 20, 2017 10:54:00 AM\\\","
- + "\\\"windowEndTime\\\":\\\"Mar 20, 2017 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ + "\\\"windowStartTime\\\":\\\"Mar 20, 2017, 10:54:00 AM\\\","
+ + "\\\"windowEndTime\\\":\\\"Mar 20, 2017, 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ "\\\"s2Id\\\":\\\"3344472187078705152\\\"}\",\"topic\":\"sample-topic\",\"logKey\":\"{"
- + "\\\"windowStartTime\\\":\\\"Mar 20, 2017 10:54:00 AM\\\","
- + "\\\"windowEndTime\\\":\\\"Mar 20, 2017 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ + "\\\"windowStartTime\\\":\\\"Mar 20, 2017, 10:54:00 AM\\\","
+ + "\\\"windowEndTime\\\":\\\"Mar 20, 2017, 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ "\\\"s2Id\\\":\\\"3344472187078705152\\\"}\"}");
}
@@ -50,8 +50,8 @@ public void shouldSerializeWhenKeyIsMissing() throws DeserializerException {
100);
String actualOutput = messageToJson.serialize(message);
assertEquals("{\"logMessage\":\"{\\\"uniqueDrivers\\\":\\\"3\\\","
- + "\\\"windowStartTime\\\":\\\"Mar 20, 2017 10:54:00 AM\\\","
- + "\\\"windowEndTime\\\":\\\"Mar 20, 2017 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ + "\\\"windowStartTime\\\":\\\"Mar 20, 2017, 10:54:00 AM\\\","
+ + "\\\"windowEndTime\\\":\\\"Mar 20, 2017, 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ "\\\"s2Id\\\":\\\"3344472187078705152\\\"}\",\"topic\":\"sample-topic\"}", actualOutput);
}
@@ -64,8 +64,8 @@ public void shouldSerializeWhenKeyIsEmptyWithTimestampsAsSimpleDateFormatWhenFla
String actualOutput = messageToJson.serialize(message);
assertEquals("{\"logMessage\":\"{\\\"uniqueDrivers\\\":\\\"3\\\","
- + "\\\"windowStartTime\\\":\\\"Mar 20, 2017 10:54:00 AM\\\","
- + "\\\"windowEndTime\\\":\\\"Mar 20, 2017 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ + "\\\"windowStartTime\\\":\\\"Mar 20, 2017, 10:54:00 AM\\\","
+ + "\\\"windowEndTime\\\":\\\"Mar 20, 2017, 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ "\\\"s2Id\\\":\\\"3344472187078705152\\\"}\",\"topic\":\"sample-topic\"}", actualOutput);
}
@@ -93,8 +93,8 @@ public void shouldWrappedSerializedJsonInArrayWithTimestampsAsSimpleDateFormatWh
String actualOutput = messageToJson.serialize(message);
assertEquals("[{\"logMessage\":\"{\\\"uniqueDrivers\\\":\\\"3\\\","
- + "\\\"windowStartTime\\\":\\\"Mar 20, 2017 10:54:00 AM\\\","
- + "\\\"windowEndTime\\\":\\\"Mar 20, 2017 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ + "\\\"windowStartTime\\\":\\\"Mar 20, 2017, 10:54:00 AM\\\","
+ + "\\\"windowEndTime\\\":\\\"Mar 20, 2017, 10:55:00 AM\\\",\\\"s2IdLevel\\\":13,\\\"vehicleType\\\":\\\"BIKE\\\","
+ "\\\"s2Id\\\":\\\"3344472187078705152\\\"}\",\"topic\":\"sample-topic\"}]", actualOutput);
}
diff --git a/src/test/java/io/odpf/firehose/sink/grpc/GrpcClientTest.java b/src/test/java/io/odpf/firehose/sink/grpc/GrpcClientTest.java
index 291d0b6b7..c7f0b0f7b 100644
--- a/src/test/java/io/odpf/firehose/sink/grpc/GrpcClientTest.java
+++ b/src/test/java/io/odpf/firehose/sink/grpc/GrpcClientTest.java
@@ -50,13 +50,13 @@ public void setup() throws IOException {
headerTestInterceptor = new HeaderTestInterceptor();
headerTestInterceptor.setHeaderKeys(HEADER_KEYS);
ServerServiceDefinition serviceDefinition = ServerInterceptors.intercept(testGrpcService.bindService(), Arrays.asList(headerTestInterceptor));
- server = ServerBuilder.forPort(5000)
+ server = ServerBuilder.forPort(5050)
.addService(serviceDefinition)
.build()
.start();
Map<String, String> config = new HashMap<>();
config.put("SINK_GRPC_SERVICE_HOST", "localhost");
- config.put("SINK_GRPC_SERVICE_PORT", "5000");
+ config.put("SINK_GRPC_SERVICE_PORT", "5050");
config.put("SINK_GRPC_METHOD_URL", "io.odpf.firehose.consumer.TestServer/TestRpcMethod");
config.put("SINK_GRPC_RESPONSE_SCHEMA_PROTO_CLASS", "io.odpf.firehose.consumer.TestGrpcResponse");
diff --git a/src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkFactoryTest.java b/src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkFactoryTest.java
index a2a6c5be8..af2f07d07 100644
--- a/src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkFactoryTest.java
+++ b/src/test/java/io/odpf/firehose/sink/grpc/GrpcSinkFactoryTest.java
@@ -43,7 +43,7 @@ public void shouldCreateChannelPoolWithHostAndPort() throws IOException, Deseria
when(testGrpcService.bindService()).thenCallRealMethod();
Server server = ServerBuilder
- .forPort(5000)
+ .forPort(5050)
.addService(testGrpcService.bindService())
.build()
.start();