Skip to content

Commit

Permalink
[Feature-15819][Task Plugin]Update the JAVA task
Browse files Browse the repository at this point in the history
  • Loading branch information
@ailiujiarui
ailiujiarui committed Sep 2, 2024
1 parent d9244ca commit 151bdfe
Show file tree
Hide file tree
Showing 8 changed files with 216 additions and 335 deletions.
17 changes: 4 additions & 13 deletions ...r-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@ private JavaConstants() {
public static final String JAVA_HOME_VAR = "${JAVA_HOME}";

/**
* this constant represents the use of the java command to run a task
* This constant represents the use of the java -jar command to run a task
**/
public static final String RUN_TYPE_JAVA = "JAVA";
public static final String RUN_TYPE_FAT_JAR = "FATJAR";

/**
* this constant represents the use of the java -jar command to run a task
* This constant represents the use of the java -cp command to run a task
**/
public static final String RUN_TYPE_JAR = "JAR";
public static final String RUN_TYPE_NORMAL_JAR = "NORMALJAR";

/**
* This constant is the Classpath or module path delimiter for different operating systems
Expand All @@ -48,13 +48,4 @@ private JavaConstants() {
**/
public static final String CLASSPATH_CURRENT_DIR = ".";

/**
* This constant is used to construct the pre-pathname of the Java source file
**/
public static final String JAVA_SOURCE_CODE_NAME_TEMPLATE = "%s/%s.java";

/**
* This constant is the regular expression to get the class name of the source file
**/
public static final String PUBLIC_CLASS_NAME_REGEX = "(.*\\s*public\\s+class\\s+)([a-zA-Z_]+[//w_]*)([.\\s\\S]*)";
}
12 changes: 3 additions & 9 deletions ...-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,13 @@
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;

import org.apache.commons.lang3.StringUtils;

import java.util.List;

import lombok.Data;

@Data
public class JavaParameters extends AbstractParameters {

/**
* origin java script
*/
private String rawScript;

/**
* run in jar file
*/
Expand Down Expand Up @@ -67,11 +60,12 @@ public class JavaParameters extends AbstractParameters {
/**
* Check that the parameters are valid
*
* @returnboolean
* @return boolean
*/
@Override
public boolean checkParameters() {
return runType != null && (StringUtils.isNotBlank(rawScript) || mainJar != null);
return runType != null && (runType.equals(JavaConstants.RUN_TYPE_FAT_JAR) ||
runType.equals(JavaConstants.RUN_TYPE_NORMAL_JAR));
}

/**
Expand Down
163 changes: 35 additions & 128 deletions ...eduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.plugin.task.java;

import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.JAVA_HOME_VAR;
import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.PUBLIC_CLASS_NAME_REGEX;

import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
Expand All @@ -27,35 +26,19 @@
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext;
import org.apache.dolphinscheduler.plugin.task.api.shell.IShellInterceptorBuilder;
import org.apache.dolphinscheduler.plugin.task.api.shell.ShellInterceptorBuilderFactory;
import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExistException;
import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException;
import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException;

import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import lombok.extern.slf4j.Slf4j;

import com.google.common.base.Preconditions;


@Slf4j
public class JavaTask extends AbstractTask {

Expand All @@ -74,10 +57,6 @@ public class JavaTask extends AbstractTask {
*/
private TaskExecutionContext taskRequest;

/**
* class name regex pattern
*/
private static final Pattern classNamePattern = Pattern.compile(PUBLIC_CLASS_NAME_REGEX);

public JavaTask(TaskExecutionContext taskRequest) {
super(taskRequest);
Expand Down Expand Up @@ -109,19 +88,20 @@ public void init() {
public void handle(TaskCallBack taskCallBack) throws TaskException {
try {
// Step 1: judge if is java or jar run type.
// Step 2 case1: the jar run type builds the command directly, adding resource to the java -jar class when
// Step 2 case1: the fat jar run type builds the command directly, adding resource to the java -jar class when
// building the command
// Step 2 case2: the normal jar run type builds the command directly, adding resource to the java -cp class when
// building the command
// Step 2 case2: the java run type, first replace the custom parameters, then compile the code, and then
// build the command will add resource
// Step 3: to run the command
String command = null;
switch (javaParameters.getRunType()) {
case JavaConstants.RUN_TYPE_JAVA:
command = buildJavaCommand();
break;
case JavaConstants.RUN_TYPE_JAR:

case JavaConstants.RUN_TYPE_FAT_JAR:
command = buildJarCommand();
break;
case JavaConstants.RUN_TYPE_NORMAL_JAR:
command = buildNormalJarCommand();
break;
default:
throw new RunTypeNotFoundException("run type is required, but it is null now.");
}
Expand Down Expand Up @@ -149,44 +129,52 @@ public void handle(TaskCallBack taskCallBack) throws TaskException {
}
}


/**
* Construct a shell command for the java Run mode
* Construct a shell command for the java -jar Run mode
*
* @return String
* @throws Exception
**/
protected String buildJavaCommand() throws Exception {
protected String buildJarCommand() {
ResourceContext resourceContext = taskRequest.getResourceContext();
String mainJarAbsolutePathInLocal = resourceContext
.getResourceItem(javaParameters.getMainJar().getResourceName())
.getResourceAbsolutePathInLocal();
StringBuilder builder = new StringBuilder();
String sourceCode = buildJavaSourceContent();
builder.append(buildJavaCompileCommand(sourceCode))
.append(";")
.append(getJavaCommandPath())
builder.append(getJavaCommandPath())
.append("java").append(" ")
.append(buildResourcePath())
.append(" ")
.append(getPublicClassName(sourceCode))
.append(" ")
.append(buildResourcePath()).append(" ")
.append("-jar").append(" ")
.append(mainJarAbsolutePathInLocal).append(" ")
.append(javaParameters.getMainArgs().trim()).append(" ")
.append(javaParameters.getJvmArgs().trim());
return builder.toString();
}


/**
* Construct a shell command for the java -jar Run mode
* Construct a shell command for the java -cp run mode
*
* @return String
**/
protected String buildJarCommand() {
protected String buildNormalJarCommand() {
ResourceContext resourceContext = taskRequest.getResourceContext();
String mainJarAbsolutePathInLocal = resourceContext
.getResourceItem(javaParameters.getMainJar().getResourceName())
String mainJarAbsolutePathInLocal = resourceContext.getResourceItem(
javaParameters.getMainJar()
.getResourceName())
.getResourceAbsolutePathInLocal();
String mainJarName=null;
try{
mainJarName = MainClassExtractor.getMainClassName(mainJarAbsolutePathInLocal);
}
catch (Exception e){
e.printStackTrace();
}
StringBuilder builder = new StringBuilder();
builder.append(getJavaCommandPath())
.append("java").append(" ")
.append(buildResourcePath()).append(" ")
.append("-jar").append(" ")
.append(mainJarAbsolutePathInLocal).append(" ")
.append(mainJarName).append(" ")
.append(javaParameters.getMainArgs().trim()).append(" ")
.append(javaParameters.getJvmArgs().trim());
return builder.toString();
Expand All @@ -207,35 +195,6 @@ public AbstractParameters getParameters() {
return javaParameters;
}

/**
* Creates a Java source file when it does not exist
*
* @param sourceCode
* @param fileName
* @return String
**/
protected void createJavaSourceFileIfNotExists(String sourceCode, String fileName) throws IOException {
log.info("tenantCode: {}, task dir:{}", taskRequest.getTenantCode(), taskRequest.getExecutePath());
if (!Files.exists(Paths.get(fileName))) {
log.info("the java source code:{}, will be write to the file: {}", fileName, sourceCode);
// write data to file
FileUtils.writeStringToFile(new File(fileName),
sourceCode,
StandardCharsets.UTF_8);
} else {
throw new JavaSourceFileExistException("java source file exists, please report an issue on official.");
}
}

/**
* Construct the full path name of the Java source file from the temporary execution path of the task
*
* @return String
**/
protected String buildJavaSourceCodeFileFullName(String publicClassName) {
return String.format(JavaConstants.JAVA_SOURCE_CODE_NAME_TEMPLATE, taskRequest.getExecutePath(),
publicClassName);
}

/**
* Construct a Classpath or module path based on isModulePath
Expand All @@ -262,46 +221,6 @@ protected String buildResourcePath() {
return builder.toString();
}

/**
* Constructs a shell command compiled from a Java source file
*
* @param sourceCode
* @return String
* @throws IOException
**/
protected String buildJavaCompileCommand(String sourceCode) throws IOException {
String publicClassName = getPublicClassName(sourceCode);
String fileName = buildJavaSourceCodeFileFullName(publicClassName);
createJavaSourceFileIfNotExists(sourceCode, fileName);

StringBuilder compilerCommand = new StringBuilder()
.append(getJavaCommandPath())
.append("javac").append(" ")
.append(buildResourcePath()).append(" ")
.append(fileName);
return compilerCommand.toString();
}

/**
* Work with Java source file content, such as replacing local variables
*
* @return String
**/
protected String buildJavaSourceContent() {
String rawJavaScript = javaParameters.getRawScript().replaceAll("\\r\\n", "\n");
// replace placeholder

Map<String, Property> paramsMap = taskRequest.getPrepareParamsMap();
if (MapUtils.isEmpty(paramsMap)) {
paramsMap = new HashMap<>();
}
if (MapUtils.isNotEmpty(taskRequest.getParamsMap())) {
paramsMap.putAll(taskRequest.getParamsMap());
}
log.info("The current java source code will begin to replace the placeholder: {}", rawJavaScript);
rawJavaScript = ParameterUtils.convertParameterPlaceholders(rawJavaScript, ParameterUtils.convert(paramsMap));
return rawJavaScript;
}

/**
* Gets the operating system absolute path to the Java command
Expand All @@ -312,17 +231,5 @@ private String getJavaCommandPath() {
return JAVA_HOME_VAR + File.separator + "bin" + File.separator;
}

/**
* Gets the public class name from the Java source file
*
* @param sourceCode
* @return String
**/
public String getPublicClassName(String sourceCode) {
Matcher matcher = classNamePattern.matcher(sourceCode);
if (!matcher.find()) {
throw new PublicClassNotFoundException("public class is not be found in source code : " + sourceCode);
}
return matcher.group(2).trim();
}

}
33 changes: 33 additions & 0 deletions ...k-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/MainClassExtractor.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.java;

import java.io.File;
import java.io.IOException;
import java.util.jar.JarFile;
import java.util.jar.Manifest;

public class MainClassExtractor {

public static String getMainClassName(String jarFilePath) throws IOException {
JarFile jarFile = new JarFile(new File(jarFilePath));
Manifest manifest = jarFile.getManifest();
String mainClassName = manifest.getMainAttributes().getValue("Main-Class");
jarFile.close();
return mainClassName;
}
}
Loading

0 comments on commit 151bdfe

Please sign in to comment.