From d67dd8553ed02218d9c4d5678f3ed580c7e4ecc9 Mon Sep 17 00:00:00 2001
From: 1996fanrui <1996fanrui@gmail.com>
Date: Thu, 20 Jul 2023 18:58:53 +0800
Subject: [PATCH] [FLIP-334] Move non-kubernetes related autoscaler classes to
flink-autoscaler module
---
Dockerfile | 2 +-
flink-autoscaler/pom.xml | 68 +++++++++++++++++++
.../autoscaler/AutoscalerFlinkMetrics.java | 24 +++++--
.../autoscaler/ScalingMetricEvaluator.java | 53 ++++++++-------
.../flink}/autoscaler/ScalingSummary.java | 8 +--
.../autoscaler/config/AutoScalerOptions.java | 10 +--
.../metrics/CollectedMetricHistory.java | 4 +-
.../autoscaler/metrics/CollectedMetrics.java | 2 +-
.../flink}/autoscaler/metrics/Edge.java | 2 +-
.../metrics/EvaluatedScalingMetric.java | 2 +-
.../autoscaler/metrics/FlinkMetric.java | 2 +-
.../autoscaler/metrics/MetricAggregator.java | 2 +-
.../autoscaler/metrics/ScalingMetric.java | 2 +-
.../autoscaler/metrics/ScalingMetrics.java | 8 +--
.../autoscaler/topology/JobTopology.java | 2 +-
.../autoscaler/topology/VertexInfo.java | 2 +-
.../utils/AutoScalerSerDeModule.java | 4 +-
.../autoscaler/utils/AutoScalerUtils.java | 17 +++--
.../ScalingMetricEvaluatorTest.java | 44 ++++++------
.../metrics/ScalingMetricsTest.java | 8 +--
.../autoscaler/utils/AutoScalerUtilsTest.java | 4 +-
.../src/test/resources/log4j2-test.properties | 26 +++++++
.../ConfigOptionsDocGenerator.java | 4 +-
flink-kubernetes-operator-autoscaler/pom.xml | 6 ++
.../operator/autoscaler/AutoScalerInfo.java | 7 +-
.../autoscaler/JobAutoScalerImpl.java | 18 ++---
.../autoscaler/JobAutoscalerFactoryImpl.java | 1 +
.../operator/autoscaler/JobVertexScaler.java | 29 ++++----
.../autoscaler/RestApiMetricsCollector.java | 2 +-
.../operator/autoscaler/ScalingExecutor.java | 19 +++---
.../autoscaler/ScalingMetricCollector.java | 16 ++---
.../autoscaler/AutoScalerInfoTest.java | 9 +--
.../autoscaler/BacklogBasedScalingTest.java | 14 ++--
.../autoscaler/JobAutoScalerImplTest.java | 12 ++--
.../operator/autoscaler/JobTopologyTest.java | 2 +-
.../autoscaler/JobVertexScalerTest.java | 8 ++-
.../MetricsCollectionAndEvaluationTest.java | 15 ++--
.../RecommendedParallelismTest.java | 15 ++--
.../RestApiMetricsCollectorTest.java | 2 +-
.../autoscaler/ScalingExecutorTest.java | 8 ++-
.../ScalingMetricCollectorTest.java | 6 +-
.../autoscaler/TestingMetricsCollector.java | 4 +-
flink-kubernetes-operator/pom.xml | 6 ++
pom.xml | 1 +
44 files changed, 317 insertions(+), 183 deletions(-)
create mode 100644 flink-autoscaler/pom.xml
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/AutoscalerFlinkMetrics.java (90%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/ScalingMetricEvaluator.java (82%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/ScalingSummary.java (85%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/config/AutoScalerOptions.java (96%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/CollectedMetricHistory.java (88%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/CollectedMetrics.java (94%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/Edge.java (94%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/EvaluatedScalingMetric.java (95%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/FlinkMetric.java (97%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/MetricAggregator.java (95%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/ScalingMetric.java (97%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/ScalingMetrics.java (97%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/topology/JobTopology.java (99%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/topology/VertexInfo.java (95%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/utils/AutoScalerSerDeModule.java (95%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/utils/AutoScalerUtils.java (82%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/ScalingMetricEvaluatorTest.java (86%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/metrics/ScalingMetricsTest.java (98%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/utils/AutoScalerUtilsTest.java (94%)
create mode 100644 flink-autoscaler/src/test/resources/log4j2-test.properties
diff --git a/Dockerfile b/Dockerfile
index 0d4a9fc8fa..548d2d3df4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,7 +23,7 @@ WORKDIR /app
COPY . .
-RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -pl flink-kubernetes-standalone,flink-kubernetes-operator-api,flink-kubernetes-operator,flink-kubernetes-operator-autoscaler,flink-kubernetes-webhook -DskipTests=$SKIP_TESTS
+RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -pl flink-kubernetes-standalone,flink-kubernetes-operator-api,flink-kubernetes-operator,flink-kubernetes-operator-autoscaler,flink-autoscaler,flink-kubernetes-webhook -DskipTests=$SKIP_TESTS
RUN cd /app/tools/license; mkdir jars; cd jars; \
cp /app/flink-kubernetes-operator/target/flink-kubernetes-operator-*-shaded.jar . && \
diff --git a/flink-autoscaler/pom.xml b/flink-autoscaler/pom.xml
new file mode 100644
index 0000000000..f945a262ac
--- /dev/null
+++ b/flink-autoscaler/pom.xml
@@ -0,0 +1,68 @@
+
+
+
+ 4.0.0
+
+
+ org.apache.flink
+ flink-kubernetes-operator-parent
+ 1.6-SNAPSHOT
+ ..
+
+
+ flink-autoscaler
+ Flink Autoscaler
+ jar
+
+
+ 2.15.0
+
+
+
+
+
+ org.apache.flink
+ flink-runtime
+ ${flink.version}
+ provided
+
+
+
+ jackson-dataformat-yaml
+ com.fasterxml.jackson.dataformat
+ ${jackson.version}
+ provided
+
+
+
+ org.projectlombok
+ lombok
+ ${lombok.version}
+ provided
+
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+ test
+
+
+
+
\ No newline at end of file
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/AutoscalerFlinkMetrics.java
similarity index 90%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/AutoscalerFlinkMetrics.java
index fab3cb3268..1a0f067132 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/AutoscalerFlinkMetrics.java
@@ -15,10 +15,10 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -37,11 +37,11 @@ public class AutoscalerFlinkMetrics {
private static final Logger LOG = LoggerFactory.getLogger(AutoscalerFlinkMetrics.class);
- final Counter numScalings;
+ private final Counter numScalings;
- final Counter numErrors;
+ private final Counter numErrors;
- final Counter numBalanced;
+ private final Counter numBalanced;
private final MetricGroup metricGroup;
@@ -54,6 +54,18 @@ public AutoscalerFlinkMetrics(MetricGroup metricGroup) {
this.metricGroup = metricGroup;
}
+ public Counter getNumScalings() {
+ return numScalings;
+ }
+
+ public Counter getNumErrors() {
+ return numErrors;
+ }
+
+ public Counter getNumBalanced() {
+ return numBalanced;
+ }
+
public void registerScalingMetrics(
Supplier