From ac736b0c9872e835d7b59b673b0c9f256db97546 Mon Sep 17 00:00:00 2001 From: brianwyka Date: Sat, 11 Dec 2021 16:22:13 -0500 Subject: [PATCH 01/35] Remote repository scanning improvements progress --- .../cli/AbstractRemoteScanCommand.java | 19 +++---- .../sourcehawk/cli/BitbucketScanCommand.java | 37 +++++++------ .../optum/sourcehawk/cli/CommandOptions.java | 11 +++- .../sourcehawk/cli/GithubScanCommand.java | 37 ++++++------- .../optum/sourcehawk/core/data/RemoteRef.java | 49 +++-------------- .../BitbucketRepositoryFileReader.java | 52 ------------------ .../GithubRepositoryFileReader.java | 54 ------------------- .../repository/LocalRepositoryFileReader.java | 6 +++ .../RemoteRepositoryFileReader.java | 51 +++++++++--------- .../core/repository/RepositoryFileReader.java | 8 +++ .../sourcehawk/exec/ExecOptionsSpec.groovy | 2 +- 11 files changed, 101 insertions(+), 225 deletions(-) delete mode 100644 core/src/main/java/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReader.java delete mode 100644 core/src/main/java/com/optum/sourcehawk/core/repository/GithubRepositoryFileReader.java diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java b/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java index daa1855..cdae269 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java @@ -52,21 +52,14 @@ public Integer call() { .isPresent(); val remoteRef = validateAndParseRemoteRef(); execOptionsBuilder.remoteRef(remoteRef); + val repositoryFileReader = createRepositoryFileReader(remoteRef); + execOptionsBuilder.repositoryFileReader(repositoryFileReader); if (StringUtils.equals(SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME, parentExecOptions.getConfigurationFileLocation()) && !configFileProvided) { - execOptionsBuilder.configurationFileLocation(constructRemoteConfigFileLocation(remoteRef)); + execOptionsBuilder.configurationFileLocation(repositoryFileReader.getAbsoluteLocation(SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME)); } - execOptionsBuilder.repositoryFileReader(createRepositoryFileReader(remoteRef)).build(); return parentCommand.call(execOptionsBuilder.build()); } - /** - * Construct the remote config file location - * - * @param remoteRef the remote reference - * @return the config file remote location - */ - protected abstract String constructRemoteConfigFileLocation(final RemoteRef remoteRef); - /** * Create the repository file reader based off the remote reference * @@ -76,11 +69,11 @@ public Integer call() { protected abstract RepositoryFileReader createRepositoryFileReader(final RemoteRef remoteRef); /** - * Get the remote reference descriptor + * Get the raw remote reference * - * @return the raw remote reference descriptor + * @return the raw remote reference */ - protected abstract Pair getRawRemoteReference(); + protected abstract Pair getRawRemoteReference(); /** * Parse the coordinates to github options diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/BitbucketScanCommand.java b/cli/src/main/java/com/optum/sourcehawk/cli/BitbucketScanCommand.java index 30c254b..c4caf44 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/BitbucketScanCommand.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/BitbucketScanCommand.java @@ -1,14 +1,15 @@ package com.optum.sourcehawk.cli; -import com.optum.sourcehawk.core.constants.SourcehawkConstants; import com.optum.sourcehawk.core.data.Pair; import com.optum.sourcehawk.core.data.RemoteRef; -import com.optum.sourcehawk.core.repository.BitbucketRepositoryFileReader; +import com.optum.sourcehawk.core.repository.RemoteRepositoryFileReader; import com.optum.sourcehawk.core.repository.RepositoryFileReader; import lombok.val; import picocli.CommandLine; -import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; import java.util.Optional; /** @@ -25,6 +26,9 @@ ) public class BitbucketScanCommand extends AbstractRemoteScanCommand { + private static final String DEFAULT_BASE_URL = "https://bitbucket.org"; + private static final String DEFAULT_REF = "main"; + /** * The Bitbucket options */ @@ -44,25 +48,24 @@ public static void main(final String... args) { /** {@inheritDoc} */ @Override protected RepositoryFileReader createRepositoryFileReader(final RemoteRef remoteRef) { - if (bitbucket.serverUrl != null) { - return new BitbucketRepositoryFileReader(bitbucket.token, bitbucket.serverUrl.toString(), remoteRef); + val rawFileUrlTemplate = Optional.ofNullable(bitbucket.serverUrl) + .map(bitbucketServerUrl -> String.format("%s/rest/api/1.0/projects/%s/repos/%s/raw/%%s?at=%s", + bitbucketServerUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef())) + .orElseGet(() -> String.format("%s/api/2.0/repositories/%s/%s/src/%s/%%s", DEFAULT_BASE_URL, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef())); + val requestProperties = new HashMap(); + requestProperties.put("Accept", "text/plain"); + if (bitbucket.token != null) { + val authScheme = Optional.ofNullable(bitbucket.authScheme) + .orElse(CommandOptions.Bitbucket.DEFAULT_AUTH_SCHEME); + requestProperties.put("Authorization", String.format("%s %s", authScheme, bitbucket.token)); } - return new BitbucketRepositoryFileReader(bitbucket.token, remoteRef); - } - - /** {@inheritDoc} */ - @Override - protected Pair getRawRemoteReference() { - return Pair.of(RemoteRef.Type.BITBUCKET, bitbucket.remoteReference); + return new RemoteRepositoryFileReader(rawFileUrlTemplate, requestProperties); } /** {@inheritDoc} */ @Override - protected String constructRemoteConfigFileLocation(final RemoteRef remoteRef) { - val bitbucketBaseUrl = Optional.ofNullable(bitbucket.serverUrl) - .map(URL::toString) - .orElseGet(RemoteRef.Type.BITBUCKET::getBaseUrl); - return BitbucketRepositoryFileReader.constructBaseUrl(remoteRef, bitbucketBaseUrl) + SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME; + protected Pair getRawRemoteReference() { + return Pair.of(bitbucket.remoteReference, DEFAULT_REF); } } diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/CommandOptions.java b/cli/src/main/java/com/optum/sourcehawk/cli/CommandOptions.java index ceba6c5..d25f175 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/CommandOptions.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/CommandOptions.java @@ -152,6 +152,15 @@ static class Bitbucket { ) String token; + @CommandLine.Option( + names = {"-a", "--auth-scheme"}, + paramLabel = "auth-scheme", + defaultValue = DEFAULT_AUTH_SCHEME, + description = "The authorization scheme to use (either Bearer or Basic). If Basic, the provided token must be base64 encoded." + ) + String authScheme; + static final String DEFAULT_AUTH_SCHEME = "Bearer"; + @CommandLine.Option( names = {"-S", "--server-url"}, paramLabel = "bitbucket-server-url", @@ -162,7 +171,7 @@ static class Bitbucket { @CommandLine.Parameters( paramLabel = REMOTE_REFERENCE_LABEL, description = "The Bitbucket remote reference - project/repo@ref combination, " - + "i.e - project/repo, project/repo@master, project/repo@v1.4, or project/repo@a6de43fa51c", + + "i.e - project/repo, project/repo@main, project/repo@v1.4, or project/repo@a6de43fa51c", arity = "1" ) String remoteReference; diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/GithubScanCommand.java b/cli/src/main/java/com/optum/sourcehawk/cli/GithubScanCommand.java index 784cf1f..b233902 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/GithubScanCommand.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/GithubScanCommand.java @@ -1,14 +1,13 @@ package com.optum.sourcehawk.cli; -import com.optum.sourcehawk.core.constants.SourcehawkConstants; import com.optum.sourcehawk.core.data.Pair; import com.optum.sourcehawk.core.data.RemoteRef; -import com.optum.sourcehawk.core.repository.GithubRepositoryFileReader; +import com.optum.sourcehawk.core.repository.RemoteRepositoryFileReader; import com.optum.sourcehawk.core.repository.RepositoryFileReader; import lombok.val; import picocli.CommandLine; -import java.net.URL; +import java.util.HashMap; import java.util.Optional; /** @@ -25,6 +24,10 @@ ) public class GithubScanCommand extends AbstractRemoteScanCommand { + private static final String DEFAULT_BASE_URL = "raw.githubusercontent.com"; + private static final String DEFAULT_REF = "main"; + private static final String AUTHORIZATION_TOKEN_PREFIX = "Bearer"; + /** * The github options */ @@ -44,28 +47,22 @@ public static void main(final String... args) { /** {@inheritDoc} */ @Override protected RepositoryFileReader createRepositoryFileReader(final RemoteRef remoteRef) { - if (github.enterpriseUrl != null) { - return new GithubRepositoryFileReader(github.token, github.enterpriseUrl.toString(), remoteRef); + val baseUrl = Optional.ofNullable(github.enterpriseUrl) + .map(githubEnterpriseUrl -> String.format("%s/raw", github.enterpriseUrl)) + .orElse(DEFAULT_BASE_URL); + val rawFileUrlTemplate = String.format("%s/%s/%s/%s/%%s", baseUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef()); + val requestProperties = new HashMap(); + requestProperties.put("Accept", "text/plain"); + if (github.token != null) { + requestProperties.put("Authorization", String.format("%s %s", AUTHORIZATION_TOKEN_PREFIX, github.token)); } - return new GithubRepositoryFileReader(github.token, remoteRef); - } - - /** {@inheritDoc} */ - @Override - protected Pair getRawRemoteReference() { - return Pair.of(RemoteRef.Type.GITHUB, github.remoteReference); + return new RemoteRepositoryFileReader(rawFileUrlTemplate, requestProperties); } /** {@inheritDoc} */ @Override - protected String constructRemoteConfigFileLocation(final RemoteRef remoteRef) { - val githubEnterpriseUrl = Optional.ofNullable(github.enterpriseUrl).map(URL::toString); - val githubRepoBaseUrl = GithubRepositoryFileReader.constructBaseUrl( - githubEnterpriseUrl.orElseGet(RemoteRef.Type.GITHUB::getBaseUrl), - githubEnterpriseUrl.isPresent(), - remoteRef - ); - return githubRepoBaseUrl + SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME; + protected Pair getRawRemoteReference() { + return Pair.of(github.remoteReference, DEFAULT_REF); } } diff --git a/core/src/main/java/com/optum/sourcehawk/core/data/RemoteRef.java b/core/src/main/java/com/optum/sourcehawk/core/data/RemoteRef.java index c17c32e..09f179e 100644 --- a/core/src/main/java/com/optum/sourcehawk/core/data/RemoteRef.java +++ b/core/src/main/java/com/optum/sourcehawk/core/data/RemoteRef.java @@ -25,13 +25,7 @@ public class RemoteRef { private static final String PARSE_ERROR_PREFIX = "Invalid remote reference"; /** - * The type of the remote reference - */ - @NonNull - Type type; - - /** - * The remote namespace, such as Github owner / organization, or Bitbucket project + * The remote namespace, such as Github owner / organization, or Bitbucket user / project */ @NonNull String namespace; @@ -51,20 +45,20 @@ public class RemoteRef { /** * Create the remote ref from the type and raw reference * - * @param type the type of the remote ref * @param rawRemoteRef the raw remote reference + * @param defaultRef the default ref to use if none provided * @return the remote reference */ - public static RemoteRef parse(final Type type, final String rawRemoteRef) { + public static RemoteRef parse(final String rawRemoteRef, final String defaultRef) { if (rawRemoteRef.indexOf(COORDINATES_DELIMITER) == -1) { - val message = String.format("%s, must contain '%s' separator between %s and repository", PARSE_ERROR_PREFIX, COORDINATES_DELIMITER, type.getNamespaceType()); + val message = String.format("%s, must contain '%s' separator between repository coordinates", PARSE_ERROR_PREFIX, COORDINATES_DELIMITER); throw new IllegalArgumentException(message); } val remoteRefBuilder = builder(); final String rawCoordinates; if (rawRemoteRef.indexOf(REF_DELIMITER) == -1) { rawCoordinates = rawRemoteRef; - remoteRefBuilder.ref(type.getDefaultBranch()); + remoteRefBuilder.ref(defaultRef); } else { val refDelimiterIndex= rawRemoteRef.indexOf(REF_DELIMITER); rawCoordinates = rawRemoteRef.substring(0, refDelimiterIndex); @@ -77,7 +71,7 @@ public static RemoteRef parse(final Type type, final String rawRemoteRef) { if (coordinates.length < 2) { throw new IllegalArgumentException(PARSE_ERROR_PREFIX + ", repository must not be empty"); } - return remoteRefBuilder.type(type) + return remoteRefBuilder .namespace(coordinates[0]) .repository(coordinates[1]) .build(); @@ -86,36 +80,7 @@ public static RemoteRef parse(final Type type, final String rawRemoteRef) { /** {@inheritDoc} */ @Override public String toString() { - return String.format("[%s] %s%s%s%s%s", type.name(), namespace, COORDINATES_DELIMITER, repository, REF_DELIMITER, ref); - } - - /** - * The type of the remote reference - * - * @author Brian Wyka - */ - @Getter - @AllArgsConstructor(access = AccessLevel.PRIVATE) - public enum Type { - - BITBUCKET("https://bitbucket.org", "master", "project"), - GITHUB("https://raw.githubusercontent.com", "main", "owner"); - - /** - * The base URL - */ - private final String baseUrl; - - /** - * The name of the default branch - */ - private final String defaultBranch; - - /** - * The namespace type - */ - private final String namespaceType; - + return String.format("%s%s%s%s%s", namespace, COORDINATES_DELIMITER, repository, REF_DELIMITER, ref); } } diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReader.java deleted file mode 100644 index ceb9ccf..0000000 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReader.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.optum.sourcehawk.core.repository; - -import com.optum.sourcehawk.core.data.RemoteRef; -import lombok.NonNull; -import lombok.val; - -/** - * A {@link RemoteRepositoryFileReader} implementation which reads files from remote Bitbucket repositories - * - * @author Brian Wyka - */ -public class BitbucketRepositoryFileReader extends RemoteRepositoryFileReader { - - /** - * The authorization token prefix - */ - private static final String AUTHORIZATION_TOKEN_PREFIX = "Bearer "; - - /** - * Constructs the Bitbucket repository file reader - * - * @param token the token - * @param bitbucketBaseUrl the Bitbucket base URL - * @param remoteRef the remote ref - */ - public BitbucketRepositoryFileReader(final String token, @NonNull final String bitbucketBaseUrl, final RemoteRef remoteRef) { - super(constructBaseUrl(remoteRef, bitbucketBaseUrl), constructRequestProperties(AUTHORIZATION_TOKEN_PREFIX, token)); - } - - /** - * Constructs an instance of this reader with the provided project, repo, and ref - * - * @param token the github token (optional) - * @param remoteRef the remote reference - */ - public BitbucketRepositoryFileReader(final String token, @NonNull final RemoteRef remoteRef) { - this(token, RemoteRef.Type.BITBUCKET.getBaseUrl(), remoteRef); - } - - /** - * Construct the base URL of the Bitbucket repository based on the provided project, repo, and ref - * - * @param remoteRef the remote reference - * @param bitbucketBaseUrl the base URL - * @return the constructed base URL with a trailing {@value #SEPARATOR} - */ - public static String constructBaseUrl(final RemoteRef remoteRef, final String bitbucketBaseUrl) { - val baseUrl = bitbucketBaseUrl.endsWith(SEPARATOR) ? bitbucketBaseUrl.substring(0, bitbucketBaseUrl.length() - 1) : bitbucketBaseUrl; - return String.format("%s/%s/%s/raw/%s/", baseUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef()); - } - -} diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/GithubRepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/GithubRepositoryFileReader.java deleted file mode 100644 index dabcf8e..0000000 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/GithubRepositoryFileReader.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.optum.sourcehawk.core.repository; - -import com.optum.sourcehawk.core.data.RemoteRef; -import lombok.NonNull; -import lombok.val; - -/** - * A {@link RemoteRepositoryFileReader} implementation which reads files from remote Github repositories - * - * @author Brian Wyka - */ -public class GithubRepositoryFileReader extends RemoteRepositoryFileReader { - - /** - * The authorization token prefix - */ - private static final String AUTHORIZATION_TOKEN_PREFIX = "token "; - - /** - * Constructs an instance of this reader with the provided Github Enterprise URL - * - * @param token the github token (optional) - * @param githubEnterpriseUrl the Github enterprise URL - * @param remoteRef the remote reference - */ - public GithubRepositoryFileReader(final String token, @NonNull final String githubEnterpriseUrl, @NonNull final RemoteRef remoteRef) { - super(constructBaseUrl(githubEnterpriseUrl, true, remoteRef), constructRequestProperties(AUTHORIZATION_TOKEN_PREFIX, token)); - } - - /** - * Constructs an instance of this reader with the provided owner, repo, and ref - * - * @param token the github token (optional) - * @param remoteRef the remote reference - */ - public GithubRepositoryFileReader(final String token, @NonNull final RemoteRef remoteRef) { - super(constructBaseUrl(RemoteRef.Type.GITHUB.getBaseUrl(), false, remoteRef), constructRequestProperties(AUTHORIZATION_TOKEN_PREFIX, token)); - } - - /** - * Construct the base URL of the Github repository based on the provided owner, repo, and ref - * - * @param githubUrl the Github URL - * @param githubEnterprise true if Github Enterprise, false otherwise - * @param remoteRef the remote reference - * @return the constructed base URL with a trailing {@value #SEPARATOR} - */ - public static String constructBaseUrl(final String githubUrl, final boolean githubEnterprise, final RemoteRef remoteRef) { - val baseUrl = githubUrl.endsWith(SEPARATOR) ? githubUrl.substring(0, githubUrl.length() - 1) : githubUrl; - val githubBaseUrl = githubEnterprise ? baseUrl + "/raw" : baseUrl; - return String.format("%s/%s/%s/%s/", githubBaseUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef()); - } - -} diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/LocalRepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/LocalRepositoryFileReader.java index 802d58d..c4f88c6 100644 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/LocalRepositoryFileReader.java +++ b/core/src/main/java/com/optum/sourcehawk/core/repository/LocalRepositoryFileReader.java @@ -59,6 +59,12 @@ public Optional read(@NonNull final String repositoryFilePath) thro return getInputStream(directory.resolve(Paths.get(repositoryFilePath))); } + /** {@inheritDoc} */ + @Override + public String getAbsoluteLocation(final String repositoryFilePath) { + return directory.resolve(Paths.get(repositoryFilePath)).toAbsolutePath().toString(); + } + /** * Get the {@link InputStream} from the {@link File} reference * diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java index 76bda54..d7a5440 100644 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java +++ b/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java @@ -16,21 +16,21 @@ /** * A remote repository file reader which treats the repository file paths relative - * to the base URL provided during construction + * to the raw file URL template provided during construction * * @author Brian Wyka */ -abstract class RemoteRepositoryFileReader implements RepositoryFileReader { +public final class RemoteRepositoryFileReader implements RepositoryFileReader { /** * URL Separator */ - protected static final String SEPARATOR = "/"; + private static final String SEPARATOR = "/"; /** - * The base URL to read from + * The raw file URL template. Takes one parameter: The path of the file in the repository */ - private final String baseUrl; + private final String rawFileUrlTemplate; /** * The required request properties @@ -45,31 +45,27 @@ abstract class RemoteRepositoryFileReader implements RepositoryFileReader { /** * Constructs an instance of this reader with the provided base URL * - * @param baseUrl the base URL + * @param rawFileUrlTemplate the raw file URL template * @param requestProperties the request properties required for connection */ - protected RemoteRepositoryFileReader(@NonNull final String baseUrl, final Map requestProperties) { - if (baseUrl.endsWith(SEPARATOR)) { - this.baseUrl = baseUrl; - } else { - this.baseUrl = baseUrl + SEPARATOR; - } + public RemoteRepositoryFileReader(@NonNull final String rawFileUrlTemplate, final Map requestProperties) { + this.rawFileUrlTemplate = rawFileUrlTemplate; this.requestProperties = requestProperties; } /** * Constructs an instance of this reader with the provided base URL * - * @param baseUrl the base URL + * @param rawFileUrlTemplate the base URL */ - protected RemoteRepositoryFileReader(@NonNull final String baseUrl) { - this(baseUrl, Collections.emptyMap()); + protected RemoteRepositoryFileReader(@NonNull final String rawFileUrlTemplate) { + this(rawFileUrlTemplate, Collections.emptyMap()); } /** {@inheritDoc} */ @Override public boolean exists(final String repositoryFilePath) throws IOException { - val absoluteUrl = constructAbsoluteUrl(baseUrl, repositoryFilePath); + val absoluteUrl = new URL(constructAbsoluteLocation(rawFileUrlTemplate, repositoryFilePath)); val absoluteUrlString = absoluteUrl.toString(); if (urlExistenceCache.containsKey(absoluteUrlString)) { return urlExistenceCache.get(absoluteUrlString); @@ -82,7 +78,7 @@ public boolean exists(final String repositoryFilePath) throws IOException { /** {@inheritDoc} */ @Override public Optional read(final String repositoryFilePath) throws IOException { - val absoluteUrl = constructAbsoluteUrl(baseUrl, repositoryFilePath); + val absoluteUrl = new URL(constructAbsoluteLocation(rawFileUrlTemplate, repositoryFilePath)); if (exists(repositoryFilePath)) { val httpUrlConnection = (HttpURLConnection) absoluteUrl.openConnection(); requestProperties.forEach(httpUrlConnection::setRequestProperty); @@ -91,6 +87,12 @@ public Optional read(final String repositoryFilePath) throws IOExce return Optional.empty(); } + /** {@inheritDoc} */ + @Override + public String getAbsoluteLocation(final String repositoryFilePath) { + return constructAbsoluteLocation(rawFileUrlTemplate, repositoryFilePath); + } + /** * Get the input stream from the {@link HttpURLConnection} * @@ -121,22 +123,21 @@ private boolean urlExists(final URL url) throws IOException { } /** - * Construct the absolute URL to the remote file + * Construct the absolute location to the remote file * - * @param baseUrl the repository base URL + * @param rawFileUrlTemplate the raw file URL template * @param repositoryFilePath the repository file path - * @return the absolute URL to the file - * @throws IOException if the URL is malformed + * @return the absolute location to the repository file */ - private static URL constructAbsoluteUrl(final String baseUrl, final String repositoryFilePath) throws IOException { + private static String constructAbsoluteLocation(final String rawFileUrlTemplate, final String repositoryFilePath) { if (repositoryFilePath.startsWith(SEPARATOR)) { - return new URL(baseUrl + repositoryFilePath.substring(1)); + return String.format(rawFileUrlTemplate, repositoryFilePath.substring(1)); } - return new URL(baseUrl + repositoryFilePath); + return String.format(rawFileUrlTemplate, repositoryFilePath); } /** - * Construct the request properties for the provided github token + * Construct the request properties for the provided authorization token * * @param authorizationPrefix the authorization request property prefix * @param authorizationToken the authorization token diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/RepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/RepositoryFileReader.java index 1fabc10..ca4c43e 100644 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/RepositoryFileReader.java +++ b/core/src/main/java/com/optum/sourcehawk/core/repository/RepositoryFileReader.java @@ -38,4 +38,12 @@ default boolean supportsGlobPatterns() { */ Optional read(final String repositoryFilePath) throws IOException; + /** + * Get a string representation of the absolute location of {@code repositoryFilePath} + * + * @param repositoryFilePath the repository file path + * @return the absolute location + */ + String getAbsoluteLocation(final String repositoryFilePath); + } diff --git a/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy b/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy index 0ca74cc..91bd09a 100644 --- a/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy +++ b/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy @@ -1,7 +1,7 @@ package com.optum.sourcehawk.exec import com.optum.sourcehawk.core.data.RemoteRef -import com.optum.sourcehawk.core.repository.GithubRepositoryFileReader + import com.optum.sourcehawk.core.repository.LocalRepositoryFileReader import com.optum.sourcehawk.core.data.Verbosity import spock.lang.Specification From 75e50549670828c4261b599b63e97abf4f43afab Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sun, 12 Dec 2021 20:03:12 -0500 Subject: [PATCH 02/35] remote-repository: Remote repository file reading improvements for Bitbucket --- .mvn/wrapper/maven-wrapper.properties | 2 +- bom/pom.xml | 2 +- cli/pom.xml | 4 +- .../cli/BitbucketScanCommandSpec.groovy | 14 +- .../cli/BitbucketScanSubCommandSpec.groovy | 18 +- .../cli/GithubScanCommandSpec.groovy | 10 +- .../flattened/sourcehawk-flattened-base.yml | 2 +- .../sourcehawk-flattened-override.yml | 2 +- cli/src/test/resources/sourcehawk-simple.yml | 22 -- core/pom.xml | 2 +- .../RemoteRepositoryFileReader.java | 40 +-- .../sourcehawk/core/data/RemoteRefSpec.groovy | 28 +- .../BitbucketRepositoryFileReaderSpec.groovy | 184 -------------- .../GithubRepositoryFileReaderSpec.groovy | 239 ------------------ .../RemoteRepositoryFileReaderSpec.groovy | 165 +++++++++--- distributions/debian/pom.xml | 2 +- distributions/docker/pom.xml | 2 +- distributions/linux/pom.xml | 2 +- distributions/pom.xml | 2 +- distributions/rpm/pom.xml | 2 +- enforcer/core/pom.xml | 2 +- enforcer/file/aot/pom.xml | 2 +- enforcer/file/common/pom.xml | 2 +- enforcer/file/core/pom.xml | 2 +- enforcer/file/docker/pom.xml | 2 +- enforcer/file/maven/pom.xml | 2 +- enforcer/file/pom.xml | 2 +- enforcer/file/registry/pom.xml | 2 +- enforcer/pom.xml | 2 +- exec/pom.xml | 4 +- .../sourcehawk/exec/scan/ScanExecutor.java | 45 +++- .../exec/scan/ScanResultFactory.java | 38 ++- .../sourcehawk/exec/ExecOptionsSpec.groovy | 9 +- .../resources/sourcehawk-flattened-base.yml | 2 +- pom.xml | 2 +- 35 files changed, 255 insertions(+), 607 deletions(-) delete mode 100644 core/src/test/groovy/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReaderSpec.groovy delete mode 100644 core/src/test/groovy/com/optum/sourcehawk/core/repository/GithubRepositoryFileReaderSpec.groovy diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index e404372..26ae971 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -1 +1 @@ -distributionUrl = https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip \ No newline at end of file +distributionUrl = https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip \ No newline at end of file diff --git a/bom/pom.xml b/bom/pom.xml index 35dbd3b..c18379b 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT sourcehawk-bom diff --git a/cli/pom.xml b/cli/pom.xml index 441e361..aeb3dee 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -8,7 +8,7 @@ sourcehawk com.optum.sourcehawk - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT sourcehawk-cli @@ -23,7 +23,7 @@ com.optum.sourcehawk.cli.Sourcehawk - 0.94 + 0.91 **/picocli/**/*.* diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanCommandSpec.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanCommandSpec.groovy index 4bb99e7..249773b 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanCommandSpec.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanCommandSpec.groovy @@ -10,11 +10,11 @@ class BitbucketScanCommandSpec extends CliBaseSpecification { def "getRawRemoteReference"() { when: - Pair rawRemoteReference = new BitbucketScanCommand(bitbucket: new CommandOptions.Bitbucket(remoteReference: "owner/repo@master")).getRawRemoteReference() + Pair rawRemoteReference = new BitbucketScanCommand(bitbucket: new CommandOptions.Bitbucket(remoteReference: "owner/repo@master")).getRawRemoteReference() then: - rawRemoteReference.left == RemoteRef.Type.BITBUCKET - rawRemoteReference.right == "owner/repo@master" + rawRemoteReference.left == "owner/repo@master" + rawRemoteReference.right == "main" } @Unroll @@ -63,16 +63,16 @@ class BitbucketScanCommandSpec extends CliBaseSpecification { def "createRepositoryFileReader"() { given: String rawReference = "project/repo@master" - BitbucketScanCommand githubScanCommand = new BitbucketScanCommand(bitbucket: new CommandOptions.Bitbucket(remoteReference: "owner/repo@main")) + BitbucketScanCommand bitbucketScanCommand = new BitbucketScanCommand(bitbucket: new CommandOptions.Bitbucket(remoteReference: "owner/repo@main")) when: - RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(RemoteRef.Type.BITBUCKET, rawReference)) + RepositoryFileReader repositoryFileReader = bitbucketScanCommand.createRepositoryFileReader(RemoteRef.parse(rawReference, "main")) then: repositoryFileReader } - def "createRepositoryFileReader - enterprise"() { + def "createRepositoryFileReader - server"() { given: String rawReference = "project/repo@master" BitbucketScanCommand githubScanCommand = new BitbucketScanCommand(bitbucket: @@ -80,7 +80,7 @@ class BitbucketScanCommandSpec extends CliBaseSpecification { ) when: - RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(RemoteRef.Type.BITBUCKET, rawReference)) + RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(rawReference, "main")) then: repositoryFileReader diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanSubCommandSpec.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanSubCommandSpec.groovy index 04f639d..b17a9d7 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanSubCommandSpec.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/BitbucketScanSubCommandSpec.groovy @@ -37,13 +37,15 @@ class BitbucketScanSubCommandSpec extends Specification { clientAndServer .when(HttpRequest.request() .withMethod("HEAD") - .withPath("/project/repo/raw/master/lombok.config"), + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok.config") + .withQueryStringParameter("at", "main"), Times.exactly(1)) .respond(HttpResponse.response().withStatusCode(200)) clientAndServer .when(HttpRequest.request() .withMethod("GET") - .withPath("/project/repo/raw/master/lombok.config") + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok.config") + .withQueryStringParameter("at", "main") .withHeader("Accept", "text/plain"), Times.exactly(2)) .respond(HttpResponse.response() @@ -65,13 +67,15 @@ class BitbucketScanSubCommandSpec extends Specification { clientAndServer .when(HttpRequest.request() .withMethod("HEAD") - .withPath("/project/repo/raw/develop/lombok2.config"), + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok2.config") + .withQueryStringParameter("at", "main"), Times.exactly(1)) .respond(HttpResponse.response().withStatusCode(200)) clientAndServer .when(HttpRequest.request() .withMethod("GET") - .withPath("/project/repo/raw/develop/lombok2.config") + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok2.config") + .withQueryStringParameter("at", "main") .withHeader("Accept", "text/plain"), Times.exactly(2)) .respond(HttpResponse.response() @@ -92,9 +96,9 @@ class BitbucketScanSubCommandSpec extends Specification { String[] args = ["bitbucket", "-S", bitbucketServerUrl, "project/repo@develop" ] clientAndServer .when(HttpRequest.request() - .withMethod("GET") - .withPath("/project/repo/raw/develop/sourcehawk.yml") - .withHeader("Accept", "text/plain"), + .withMethod("HEAD") + .withPath("/rest/api/1.0/projects/project/repos/repo/raw/lombok.config") + .withQueryStringParameter("at", "main"), Times.exactly(1)) .respond(HttpResponse.notFoundResponse()) diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/GithubScanCommandSpec.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/GithubScanCommandSpec.groovy index 87cfe86..6c84afe 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/GithubScanCommandSpec.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/GithubScanCommandSpec.groovy @@ -10,11 +10,11 @@ class GithubScanCommandSpec extends CliBaseSpecification { def "getRawRemoteReference"() { when: - Pair rawRemoteReference = new GithubScanCommand(github: new CommandOptions.Github(remoteReference: "owner/repo@main")).getRawRemoteReference() + Pair rawRemoteReference = new GithubScanCommand(github: new CommandOptions.Github(remoteReference: "owner/repo@main")).getRawRemoteReference() then: - rawRemoteReference.left == RemoteRef.Type.GITHUB - rawRemoteReference.right == "owner/repo@main" + rawRemoteReference.left == "owner/repo@main" + rawRemoteReference.right == "main" } @Unroll @@ -66,7 +66,7 @@ class GithubScanCommandSpec extends CliBaseSpecification { GithubScanCommand githubScanCommand = new GithubScanCommand(github: new CommandOptions.Github(remoteReference: "owner/repo@main")) when: - RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(RemoteRef.Type.GITHUB, rawReference)) + RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(rawReference, "main")) then: repositoryFileReader @@ -80,7 +80,7 @@ class GithubScanCommandSpec extends CliBaseSpecification { ) when: - RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(RemoteRef.Type.GITHUB, rawReference)) + RepositoryFileReader repositoryFileReader = githubScanCommand.createRepositoryFileReader(RemoteRef.parse(rawReference, "main")) then: repositoryFileReader diff --git a/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml b/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml index b250224..efcca44 100644 --- a/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml +++ b/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml @@ -53,4 +53,4 @@ file-protocols: enforcers: - enforcer: ".common.StringPropertyEquals" property-name: "distributionUrl" - expected-property-value: "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip" + expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip" diff --git a/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml b/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml index 0f6a243..8c715e9 100644 --- a/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml +++ b/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml @@ -72,4 +72,4 @@ file-protocols: enforcers: - enforcer: ".common.StringPropertyEquals" property-name: "distributionUrl" - expected-property-value: "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip" + expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip" diff --git a/cli/src/test/resources/sourcehawk-simple.yml b/cli/src/test/resources/sourcehawk-simple.yml index 908c760..8bf174b 100644 --- a/cli/src/test/resources/sourcehawk-simple.yml +++ b/cli/src/test/resources/sourcehawk-simple.yml @@ -8,28 +8,6 @@ file-protocols: repository-path: README.md required: true severity: WARNING - - name: Lombok - repository-path: lombok.config - required: true - severity: WARNING - enforcers: - - enforcer: .common.StringPropertyEquals - property-name: config.stopBubbling - expected-property-value: false - - enforcer: .common.StringPropertyEquals - property-name: lombok.addLombokGeneratedAnnotation - expected-property-value: false - - name: Lombok - repository-path: lombok.config - required: true - severity: WARNING - enforcers: - - enforcer: .common.StringPropertyEquals - property-name: config.stopBubbling - expected-property-value: false - - enforcer: .common.StringPropertyEquals - property-name: lombok.addLombokGeneratedAnnotation - expected-property-value: false - name: Lombok repository-path: lombok.config required: true diff --git a/core/pom.xml b/core/pom.xml index aa79050..a840223 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java b/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java index d7a5440..20c49b8 100644 --- a/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java +++ b/core/src/main/java/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReader.java @@ -1,18 +1,15 @@ package com.optum.sourcehawk.core.repository; -import com.optum.sourcehawk.core.utils.StringUtils; -import lombok.NonNull; -import lombok.val; - import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import lombok.NonNull; +import lombok.val; /** * A remote repository file reader which treats the repository file paths relative @@ -48,20 +45,11 @@ public final class RemoteRepositoryFileReader implements RepositoryFileReader { * @param rawFileUrlTemplate the raw file URL template * @param requestProperties the request properties required for connection */ - public RemoteRepositoryFileReader(@NonNull final String rawFileUrlTemplate, final Map requestProperties) { + public RemoteRepositoryFileReader(@NonNull final String rawFileUrlTemplate, @NonNull final Map requestProperties) { this.rawFileUrlTemplate = rawFileUrlTemplate; this.requestProperties = requestProperties; } - /** - * Constructs an instance of this reader with the provided base URL - * - * @param rawFileUrlTemplate the base URL - */ - protected RemoteRepositoryFileReader(@NonNull final String rawFileUrlTemplate) { - this(rawFileUrlTemplate, Collections.emptyMap()); - } - /** {@inheritDoc} */ @Override public boolean exists(final String repositoryFilePath) throws IOException { @@ -119,7 +107,11 @@ private boolean urlExists(final URL url) throws IOException { val httpUrlConnection = (HttpURLConnection) url.openConnection(); httpUrlConnection.setRequestMethod("HEAD"); requestProperties.forEach(httpUrlConnection::setRequestProperty); - return httpUrlConnection.getResponseCode() == HttpURLConnection.HTTP_OK; + val httpResponseCode = httpUrlConnection.getResponseCode(); + if (httpResponseCode != HttpURLConnection.HTTP_OK) { + System.err.println("HTTP Request to " + url + " returned response code " + httpResponseCode); // FIXME + } + return httpResponseCode == HttpURLConnection.HTTP_OK; } /** @@ -136,20 +128,4 @@ private static String constructAbsoluteLocation(final String rawFileUrlTemplate, return String.format(rawFileUrlTemplate, repositoryFilePath); } - /** - * Construct the request properties for the provided authorization token - * - * @param authorizationPrefix the authorization request property prefix - * @param authorizationToken the authorization token - * @return the request properties - */ - protected static Map constructRequestProperties(final String authorizationPrefix, final String authorizationToken) { - val requestProperties = new HashMap(); - requestProperties.put("Accept", "text/plain"); - if (StringUtils.isNotBlankOrEmpty(authorizationToken)) { - requestProperties.put("Authorization", authorizationPrefix + authorizationToken); - } - return requestProperties; - } - } diff --git a/core/src/test/groovy/com/optum/sourcehawk/core/data/RemoteRefSpec.groovy b/core/src/test/groovy/com/optum/sourcehawk/core/data/RemoteRefSpec.groovy index 5d7e567..7b26ae4 100644 --- a/core/src/test/groovy/com/optum/sourcehawk/core/data/RemoteRefSpec.groovy +++ b/core/src/test/groovy/com/optum/sourcehawk/core/data/RemoteRefSpec.groovy @@ -7,14 +7,12 @@ class RemoteRefSpec extends Specification { def "builder and toString"() { given: - RemoteRef.Type type = RemoteRef.Type.GITHUB String namespace = "namespace" String repository = "repository" String ref = "ref" when: RemoteRef remoteRef = RemoteRef.builder() - .type(type) .namespace(namespace) .repository(repository) .ref(ref) @@ -22,32 +20,29 @@ class RemoteRefSpec extends Specification { then: remoteRef - remoteRef.type == type remoteRef.namespace == namespace remoteRef.repository == repository remoteRef.ref == ref and: - remoteRef.toString() == "[GITHUB] namespace/repository@ref" + remoteRef.toString() == "namespace/repository@ref" } def "parse"() { given: - RemoteRef.Type type = RemoteRef.Type.GITHUB String rawReference = "namespace/repository@ref" when: - RemoteRef remoteRef = RemoteRef.parse(type, rawReference) + RemoteRef remoteRef = RemoteRef.parse(rawReference, "main") then: remoteRef - remoteRef.type == type remoteRef.namespace == "namespace" remoteRef.repository == "repository" remoteRef.ref == "ref" and: - remoteRef.toString() == "[GITHUB] ${rawReference}" + remoteRef.toString() == rawReference } @Unroll @@ -56,31 +51,22 @@ class RemoteRefSpec extends Specification { String rawReference = "namespace/repository" when: - RemoteRef remoteRef = RemoteRef.parse(type, rawReference) + RemoteRef remoteRef = RemoteRef.parse(rawReference, "main") then: remoteRef - remoteRef.type == type remoteRef.namespace == "namespace" remoteRef.repository == "repository" - remoteRef.ref == expected + remoteRef.ref == "main" and: - remoteRef.toString() == "[${type.name()}] ${rawReference}@${expected}" - - where: - type | expected - RemoteRef.Type.GITHUB | "main" - RemoteRef.Type.BITBUCKET | "master" + remoteRef.toString() == "${rawReference}@main" } @Unroll def "parse - invalid (throws IllegalArgumentException)"() { - given: - RemoteRef.Type type = RemoteRef.Type.GITHUB - when: - RemoteRef.parse(type, rawReference) + RemoteRef.parse(rawReference, "main") then: thrown(IllegalArgumentException) diff --git a/core/src/test/groovy/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReaderSpec.groovy b/core/src/test/groovy/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReaderSpec.groovy deleted file mode 100644 index 6aef8f3..0000000 --- a/core/src/test/groovy/com/optum/sourcehawk/core/repository/BitbucketRepositoryFileReaderSpec.groovy +++ /dev/null @@ -1,184 +0,0 @@ -package com.optum.sourcehawk.core.repository - -import com.optum.sourcehawk.core.data.RemoteRef -import org.mockserver.configuration.ConfigurationProperties -import org.mockserver.integration.ClientAndServer -import org.mockserver.matchers.Times -import org.mockserver.model.HttpRequest -import org.mockserver.model.HttpResponse -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.lang.Specification - -class BitbucketRepositoryFileReaderSpec extends Specification { - - @Shared - @AutoCleanup - ClientAndServer clientAndServer - - @Shared - String baseUrl - - def setupSpec() { - clientAndServer = ClientAndServer.startClientAndServer("http://127.0.0.1", 8122) - ConfigurationProperties.logLevel("WARN") - baseUrl = "${clientAndServer.remoteAddress.hostString}:${clientAndServer.port}" - } - - def setup() { - clientAndServer.reset() - } - - def "constructors"() { - expect: - new BitbucketRepositoryFileReader(null, RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - new BitbucketRepositoryFileReader("abc", "https://bitbucket.example.com/", RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - } - - def "supportsGlobPatterns"() { - given: - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - - when: - boolean supportsGlobPatterns = reader.supportsGlobPatterns() - - then: - !supportsGlobPatterns - } - - def "exists - found"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, baseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.response().withStatusCode(200)) - - when: - boolean exists = reader.exists("README.md") - - then: - exists - } - - def "exists - not found"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, baseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.notFoundResponse()) - - when: - boolean exists = reader.exists("README.md") - - then: - !exists - } - - def "read - found"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, baseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.response().withStatusCode(200)) - clientAndServer - .when(HttpRequest.request() - .withMethod("GET") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(2)) - .respond(HttpResponse.response().withStatusCode(200).withBody("# Title".bytes)) - - when: - Optional inputStreamOptional = reader.read("README.md") - - then: - inputStreamOptional - inputStreamOptional.isPresent() - - when: - inputStreamOptional = reader.read("/README.md") - - then: - inputStreamOptional - inputStreamOptional.isPresent() - } - - def "read - not found"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - BitbucketRepositoryFileReader reader = new BitbucketRepositoryFileReader(null, baseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/project/repo/raw/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.notFoundResponse()) - - when: - Optional inputStreamOptional = reader.read("README.md") - - then: - !inputStreamOptional - } - - def "constructBaseUrl"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master") - - when: - String baseUrl = BitbucketRepositoryFileReader.constructBaseUrl(remoteRef, RemoteRef.Type.BITBUCKET.baseUrl) - - then: - baseUrl == "https://bitbucket.org/project/repo/raw/master/" - } - - def "constructor - null parameter"() { - when: - new BitbucketRepositoryFileReader("abc", null) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader(null, null) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader("abc", null, RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader(null, null, RemoteRef.parse(RemoteRef.Type.BITBUCKET, "project/repo@master")) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader("abc", null, null) - - then: - thrown(NullPointerException) - - when: - new BitbucketRepositoryFileReader(null, null, null) - - then: - thrown(NullPointerException) - } - -} diff --git a/core/src/test/groovy/com/optum/sourcehawk/core/repository/GithubRepositoryFileReaderSpec.groovy b/core/src/test/groovy/com/optum/sourcehawk/core/repository/GithubRepositoryFileReaderSpec.groovy deleted file mode 100644 index d2e3e37..0000000 --- a/core/src/test/groovy/com/optum/sourcehawk/core/repository/GithubRepositoryFileReaderSpec.groovy +++ /dev/null @@ -1,239 +0,0 @@ -package com.optum.sourcehawk.core.repository - -import com.optum.sourcehawk.core.data.RemoteRef -import org.mockserver.configuration.ConfigurationProperties -import org.mockserver.integration.ClientAndServer -import org.mockserver.matchers.Times -import org.mockserver.model.HttpRequest -import org.mockserver.model.HttpResponse -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.lang.Specification -import spock.lang.Unroll - -class GithubRepositoryFileReaderSpec extends Specification { - - @Shared - @AutoCleanup - ClientAndServer clientAndServer - - @Shared - String enterpriseUrl - - def setupSpec() { - clientAndServer = ClientAndServer.startClientAndServer("http://127.0.0.1", 8121) - ConfigurationProperties.logLevel("WARN") - enterpriseUrl = "${clientAndServer.remoteAddress.hostString}:${clientAndServer.port}" - } - - def setup() { - clientAndServer.reset() - } - - def "supportsGlobPatterns"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, enterpriseUrl, remoteRef) - - when: - boolean supportsGlobPatterns = githubRepositoryFileReader.supportsGlobPatterns() - - then: - !supportsGlobPatterns - } - - def "exists (found)"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, enterpriseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/raw/owner/repo/main/README.md"), - Times.exactly(1)) - .respond(HttpResponse.response().withStatusCode(200)) - - when: - boolean exists = githubRepositoryFileReader.exists("README.md") - - then: - exists - } - - def "exists (not found)"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@nope") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, enterpriseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/raw/owner/repo/main/README.md"), - Times.exactly(1)) - .respond(HttpResponse.notFoundResponse()) - - when: - boolean exists = githubRepositoryFileReader.exists("README.md") - - then: - !exists - } - - def "read (found)"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, enterpriseUrl, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/raw/owner/repo/main/README.md"), - Times.exactly(1)) - .respond(HttpResponse.response().withStatusCode(200)) - clientAndServer - .when(HttpRequest.request() - .withMethod("GET") - .withPath("/raw/owner/repo/main/README.md"), - Times.exactly(2)) - .respond(HttpResponse.response().withStatusCode(200).withBody("# Title".bytes)) - - when: - Optional inputStream = githubRepositoryFileReader.read("README.md") - - then: - inputStream - inputStream.isPresent() - - when: - inputStream = githubRepositoryFileReader.read("/README.md") - - then: - inputStream - inputStream.isPresent() - } - - def "read (not found)"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@master") - GithubRepositoryFileReader githubRepositoryFileReader = new GithubRepositoryFileReader(null, remoteRef) - clientAndServer - .when(HttpRequest.request() - .withMethod("HEAD") - .withPath("/raw/owner/repo/master/README.md"), - Times.exactly(1)) - .respond(HttpResponse.notFoundResponse()) - - when: - Optional inputStream = githubRepositoryFileReader.read("README.md") - - then: - !inputStream - !inputStream.isPresent() - } - - def "constructBaseUrl - public github"() { - given: - String githubUrl = "https://raw.githubusercontent.com" - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - - when: - String baseUrl = GithubRepositoryFileReader.constructBaseUrl(githubUrl, false, remoteRef) - - then: - baseUrl == "https://raw.githubusercontent.com/owner/repo/main/" - } - - @Unroll - def "constructBaseUrl - enterprise github"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - - when: - String baseUrl = GithubRepositoryFileReader.constructBaseUrl(githubUrl, true, remoteRef) - - then: - baseUrl == "https://github.example.com/raw/owner/repo/main/" - - where: - githubUrl << ["https://github.example.com", "https://github.example.com/"] - } - - def "constructRequestProperties"() { - when: - Map requestProperties = GithubRepositoryFileReader.constructRequestProperties("token ", "abc") - - then: - requestProperties - requestProperties.size() == 2 - requestProperties["Accept"] == "text/plain" - requestProperties["Authorization"] == "token abc" - } - - def "constructRequestProperties - null"() { - when: - Map requestProperties = GithubRepositoryFileReader.constructRequestProperties("token ", null) - - then: - requestProperties - requestProperties.size() == 1 - requestProperties["Accept"] == "text/plain" - } - - def "constructor - enterprise"() { - given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") - - expect: - new GithubRepositoryFileReader(null, "https://github.example.com", remoteRef) - new GithubRepositoryFileReader("abc", "https://github.example.com", remoteRef) - } - - def "constructors - null parameter"() { - when: - new GithubRepositoryFileReader(null, null, null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader("abc", null, null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader("abc", null, RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main")) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader("abc", "https://github.example.com", null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader(null, "https://github.example.com", null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader(null, null, RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main")) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader("abc", null) - - then: - thrown(NullPointerException) - - when: - new GithubRepositoryFileReader(null, null) - - then: - thrown(NullPointerException) - } - -} diff --git a/core/src/test/groovy/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReaderSpec.groovy b/core/src/test/groovy/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReaderSpec.groovy index 51592b5..ef755dc 100644 --- a/core/src/test/groovy/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReaderSpec.groovy +++ b/core/src/test/groovy/com/optum/sourcehawk/core/repository/RemoteRepositoryFileReaderSpec.groovy @@ -1,71 +1,168 @@ package com.optum.sourcehawk.core.repository -import spock.lang.Specification +import org.mockserver.configuration.ConfigurationProperties +import org.mockserver.integration.ClientAndServer +import org.mockserver.matchers.Times +import org.mockserver.model.HttpRequest +import org.mockserver.model.HttpResponse +import spock.lang.AutoCleanup +import spock.lang.Shared +import spock.lang.Specification class RemoteRepositoryFileReaderSpec extends Specification { - def "constructor"() { - expect: - new GenericRemoteRepositoryFileReader() - new TrailingSlashRemoteRepositoryFileReader() + @Shared + @AutoCleanup + ClientAndServer clientAndServer + + @Shared + String baseUrl + + def setupSpec() { + clientAndServer = ClientAndServer.startClientAndServer("http://127.0.0.1", 8122) + ConfigurationProperties.logLevel("WARN") + baseUrl = "${clientAndServer.remoteAddress.hostString}:${clientAndServer.port}" } - def "constructor - null argument"() { + def "supportsGlobPatterns"() { + given: + RepositoryFileReader reader = new RemoteRepositoryFileReader("rawFileUrlTemplate", Collections.emptyMap()) + when: - new InvalidRemoteRepositoryFileReader() + boolean supportsGlobPatterns = reader.supportsGlobPatterns() then: - thrown(NullPointerException) + !supportsGlobPatterns + } + + def "exists - found"() { + given: + String rawFileUrlTemplate = "$baseUrl/project/repo/raw/master/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + clientAndServer + .when(HttpRequest.request() + .withMethod("HEAD") + .withPath("/project/repo/raw/master/README.md"), + Times.exactly(1)) + .respond(HttpResponse.response().withStatusCode(200)) when: - new InvalidRemoteRepositoryFileReader2() + boolean exists = reader.exists("README.md") then: - thrown(NullPointerException) + exists } - def "getInputStream - not found"() { + def "exists - not found"() { given: - HttpURLConnection mockHttpUrlConnection = Mock() + String rawFileUrlTemplate = "$baseUrl/project/repo/raw/master/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + clientAndServer + .when(HttpRequest.request() + .withMethod("HEAD") + .withPath("/project/repo/raw/master/README.md"), + Times.exactly(1)) + .respond(HttpResponse.notFoundResponse()) when: - Optional inputStreamOptional = RemoteRepositoryFileReader.getInputStream(mockHttpUrlConnection) + boolean exists = reader.exists("README.md") then: - 1 * mockHttpUrlConnection.getInputStream() >> { throw new FileNotFoundException("404") } - 0 * _ - - and: - !inputStreamOptional.isPresent() + !exists } - private static class GenericRemoteRepositoryFileReader extends RemoteRepositoryFileReader { + def "read - found"() { + given: + String rawFileUrlTemplate = "$baseUrl/raw/project/repo/main/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + clientAndServer + .when(HttpRequest.request() + .withMethod("HEAD") + .withPath("/raw/project/repo/main/README.md"), + Times.exactly(1)) + .respond(HttpResponse.response().withStatusCode(200)) + clientAndServer + .when(HttpRequest.request() + .withMethod("GET") + .withPath("/raw/project/repo/main/README.md"), + Times.exactly(2)) + .respond(HttpResponse.response().withStatusCode(200).withBody("# Title".bytes)) + + when: + Optional inputStreamOptional = reader.read("README.md") + + then: + inputStreamOptional + inputStreamOptional.isPresent() + + when: + inputStreamOptional = reader.read("/README.md") - protected GenericRemoteRepositoryFileReader() { - super("https://optum.github.io") - } + then: + inputStreamOptional + inputStreamOptional.isPresent() } - private static class TrailingSlashRemoteRepositoryFileReader extends RemoteRepositoryFileReader { + def "read - not found"() { + given: + String rawFileUrlTemplate = "$baseUrl/raw/project/repo/main/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + clientAndServer + .when(HttpRequest.request() + .withMethod("HEAD") + .withPath("/raw/project/repo/main/README.md"), + Times.exactly(1)) + .respond(HttpResponse.notFoundResponse()) + + when: + Optional inputStreamOptional = reader.read("README.md") - protected TrailingSlashRemoteRepositoryFileReader() { - super("https://optum.github.io/") - } + then: + !inputStreamOptional.isPresent() } - private static class InvalidRemoteRepositoryFileReader extends RemoteRepositoryFileReader { + def "getAbsoluteLocation"() { + given: + String rawFileUrlTemplate = "$baseUrl/raw/project/repo/main/%s" + RepositoryFileReader reader = new RemoteRepositoryFileReader(rawFileUrlTemplate, Collections.emptyMap()) + String repositoryFilePath = "README.md" + + when: + String absoluteLocation = reader.getAbsoluteLocation(repositoryFilePath) + + then: + absoluteLocation + absoluteLocation == "$baseUrl/raw/project/repo/main/README.md" + + + when: + repositoryFilePath = "/path/to/file.txt" + absoluteLocation = reader.getAbsoluteLocation(repositoryFilePath) - protected InvalidRemoteRepositoryFileReader() { - super(null) - } + then: + absoluteLocation + absoluteLocation == "$baseUrl/raw/project/repo/main/path/to/file.txt" } - private static class InvalidRemoteRepositoryFileReader2 extends RemoteRepositoryFileReader { + def "constructor - null parameter"() { + when: + new RemoteRepositoryFileReader("abc", null) + + then: + thrown(NullPointerException) + + when: + new RemoteRepositoryFileReader(null, Collections.emptyMap()) + + then: + thrown(NullPointerException) + + when: + new RemoteRepositoryFileReader(null, null) - protected InvalidRemoteRepositoryFileReader2() { - super(null, Collections.emptyMap()) - } + then: + thrown(NullPointerException) } } \ No newline at end of file diff --git a/distributions/debian/pom.xml b/distributions/debian/pom.xml index 7c27965..359aeec 100644 --- a/distributions/debian/pom.xml +++ b/distributions/debian/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-dist - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/distributions/docker/pom.xml b/distributions/docker/pom.xml index 4412275..bb59e6f 100644 --- a/distributions/docker/pom.xml +++ b/distributions/docker/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-dist - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index 561a2ca..be5e6dc 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-dist - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/distributions/pom.xml b/distributions/pom.xml index a5e8fb1..e1a410e 100644 --- a/distributions/pom.xml +++ b/distributions/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/distributions/rpm/pom.xml b/distributions/rpm/pom.xml index 4a63270..10bac32 100644 --- a/distributions/rpm/pom.xml +++ b/distributions/rpm/pom.xml @@ -8,7 +8,7 @@ sourcehawk-dist com.optum.sourcehawk - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT sourcehawk-dist-rpm diff --git a/enforcer/core/pom.xml b/enforcer/core/pom.xml index 11ef76f..93e09a2 100644 --- a/enforcer/core/pom.xml +++ b/enforcer/core/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/enforcer/file/aot/pom.xml b/enforcer/file/aot/pom.xml index 28d027c..7c233bf 100644 --- a/enforcer/file/aot/pom.xml +++ b/enforcer/file/aot/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/enforcer/file/common/pom.xml b/enforcer/file/common/pom.xml index aab36b7..8d0ef8a 100644 --- a/enforcer/file/common/pom.xml +++ b/enforcer/file/common/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/enforcer/file/core/pom.xml b/enforcer/file/core/pom.xml index 082fa20..cd74cb7 100644 --- a/enforcer/file/core/pom.xml +++ b/enforcer/file/core/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/enforcer/file/docker/pom.xml b/enforcer/file/docker/pom.xml index bda2632..988a7bc 100644 --- a/enforcer/file/docker/pom.xml +++ b/enforcer/file/docker/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/enforcer/file/maven/pom.xml b/enforcer/file/maven/pom.xml index 06e1453..fba5aea 100644 --- a/enforcer/file/maven/pom.xml +++ b/enforcer/file/maven/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/enforcer/file/pom.xml b/enforcer/file/pom.xml index 5ef024b..1852619 100644 --- a/enforcer/file/pom.xml +++ b/enforcer/file/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/enforcer/file/registry/pom.xml b/enforcer/file/registry/pom.xml index 71ac145..bf8c359 100644 --- a/enforcer/file/registry/pom.xml +++ b/enforcer/file/registry/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/enforcer/pom.xml b/enforcer/pom.xml index 2ecf225..6254ec3 100644 --- a/enforcer/pom.xml +++ b/enforcer/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml diff --git a/exec/pom.xml b/exec/pom.xml index a517011..1033150 100644 --- a/exec/pom.xml +++ b/exec/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT ../pom.xml @@ -19,7 +19,7 @@ - 0.88 + 0.87 diff --git a/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanExecutor.java b/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanExecutor.java index edfbdce..7c8e96f 100644 --- a/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanExecutor.java +++ b/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanExecutor.java @@ -1,19 +1,15 @@ package com.optum.sourcehawk.exec.scan; import com.optum.sourcehawk.core.configuration.SourcehawkConfiguration; +import com.optum.sourcehawk.core.data.Severity; import com.optum.sourcehawk.core.protocol.file.FileProtocol; import com.optum.sourcehawk.core.result.ScanResult; -import com.optum.sourcehawk.core.data.Severity; import com.optum.sourcehawk.core.utils.CollectionUtils; import com.optum.sourcehawk.core.utils.FileUtils; import com.optum.sourcehawk.core.utils.Try; import com.optum.sourcehawk.enforcer.file.FileEnforcer; import com.optum.sourcehawk.exec.ConfigurationReader; import com.optum.sourcehawk.exec.ExecOptions; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.val; - import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; @@ -21,6 +17,9 @@ import java.util.Collection; import java.util.Collections; import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.val; /** * Entry point into executing scans @@ -117,7 +116,11 @@ private static ScanResult enforceFileProtocol(final ExecOptions execOptions, fin if (execOptions.getRepositoryFileReader().supportsGlobPatterns() && FileUtils.isGlobPattern(fileProtocol.getRepositoryPath())) { fileProtocolScanResults.addAll(executeFileEnforcerOnGlob(execOptions, fileProtocol, fileEnforcer)); } else { - fileProtocolScanResults.add(executeFileEnforcer(execOptions, fileProtocol.getRepositoryPath(), fileProtocol.getSeverity(), fileEnforcer)); + val scanResult = executeFileEnforcer(execOptions, fileProtocol.getRepositoryPath(), fileProtocol, fileEnforcer); + fileProtocolScanResults.add(scanResult); + if (isScanResultFileNotFound(scanResult)) { + break; + } } } return fileProtocolScanResults.stream() @@ -145,7 +148,7 @@ private static Collection executeFileEnforcerOnGlob(final ExecOption } val fileEnforcerScanResults = new ArrayList(repositoryPaths.size()); for (val repositoryPath : repositoryPaths) { - fileEnforcerScanResults.add(executeFileEnforcer(execOptions, repositoryPath, fileProtocol.getSeverity(), fileEnforcer)); + fileEnforcerScanResults.add(executeFileEnforcer(execOptions, repositoryPath, fileProtocol, fileEnforcer)); } return fileEnforcerScanResults; } @@ -154,19 +157,33 @@ private static Collection executeFileEnforcerOnGlob(final ExecOption * Execute the file enforcer to produce the scan result * * @param execOptions the exec options - * @param repositoryFilePath the repository file path - * @param severity the severity of the file protocol + * @param repositoryPath the repository path + * @param fileProtocol the file protocol * @param fileEnforcer the file enforcer to execute * @return the scan result * @throws IOException if any error occurs accessing the file or executing enforcer */ - private static ScanResult executeFileEnforcer(final ExecOptions execOptions, final String repositoryFilePath, final String severity, - final FileEnforcer fileEnforcer) throws IOException { - try (val fileInputStream = execOptions.getRepositoryFileReader().read(repositoryFilePath) - .orElseThrow(() -> new IOException(String.format("File not found: %s", repositoryFilePath)))) { + private static ScanResult executeFileEnforcer(final ExecOptions execOptions, final String repositoryPath, final FileProtocol fileProtocol, + final FileEnforcer fileEnforcer) throws IOException { + val fileInputStreamOptional = execOptions.getRepositoryFileReader().read(repositoryPath); + if (!fileInputStreamOptional.isPresent()) { + return ScanResultFactory.fileNotFound(execOptions, repositoryPath, fileProtocol.getSeverity()); + } + try (val fileInputStream = fileInputStreamOptional.get()) { val enforcerResult = fileEnforcer.enforce(fileInputStream); - return ScanResultFactory.enforcerResult(execOptions, repositoryFilePath, Severity.parse(severity), enforcerResult); + return ScanResultFactory.enforcerResult(execOptions, repositoryPath, Severity.parse(fileProtocol.getSeverity()), enforcerResult); } } + /** + * Determine if the {@link ScanResult} is because the file was not found + * + * @param scanResult the scan result + * @return true if because of file not being found, false otherwise + */ + private static boolean isScanResultFileNotFound(final ScanResult scanResult) { + return scanResult.getFormattedMessages().size() == 1 + && scanResult.getFormattedMessages().stream().anyMatch(message -> message.contains("File not found")); + } + } diff --git a/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanResultFactory.java b/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanResultFactory.java index 32c3245..9bf3099 100644 --- a/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanResultFactory.java +++ b/exec/src/main/java/com/optum/sourcehawk/exec/scan/ScanResultFactory.java @@ -5,13 +5,12 @@ import com.optum.sourcehawk.core.result.ScanResult; import com.optum.sourcehawk.enforcer.EnforcerResult; import com.optum.sourcehawk.exec.ExecOptions; -import lombok.experimental.UtilityClass; -import lombok.val; - import java.util.ArrayList; import java.util.Collections; import java.util.Optional; import java.util.function.IntConsumer; +import lombok.experimental.UtilityClass; +import lombok.val; /** * A factory for creating instances of {@link ScanResult} @@ -95,18 +94,31 @@ public ScanResult globalError(final Throwable throwable) { * @return the file not found scan result */ public ScanResult fileNotFound(final ExecOptions execOptions, final FileProtocol fileProtocol) { - val severity = Severity.parse(fileProtocol.getSeverity()); + return fileNotFound(execOptions, fileProtocol.getRepositoryPath(), fileProtocol.getSeverity()); + } + + /** + * Generate a scan result for situations where the file is not found + * + * @param execOptions the exec options + * @param repositoryPath the repository path + * @param fileProtocolSeverity the file protocol severity + * @return the file not found scan result + */ + public ScanResult fileNotFound(final ExecOptions execOptions, final String repositoryPath, final String fileProtocolSeverity) { + val severity = Severity.parse(fileProtocolSeverity); val messageDescriptor = ScanResult.MessageDescriptor.builder() - .severity(fileProtocol.getSeverity()) - .repositoryPath(fileProtocol.getRepositoryPath()) - .message("File not found") - .build(); + .severity(fileProtocolSeverity) + .repositoryPath(repositoryPath) + .message("File not found") + .build(); val scanResultBuilder = ScanResult.builder() - .passed(Severity.WARNING.equals(severity) && !execOptions.isFailOnWarnings()) - .messages(Collections.singletonMap(fileProtocol.getRepositoryPath(), Collections.singleton(messageDescriptor))) - .formattedMessages(Collections.singleton(messageDescriptor.toString())); - return acceptCount(scanResultBuilder, Severity.parse(fileProtocol.getSeverity()), 1) - .build(); + + .passed(Severity.WARNING.equals(severity) && !execOptions.isFailOnWarnings()) + .messages(Collections.singletonMap(repositoryPath, Collections.singleton(messageDescriptor))) + .formattedMessages(Collections.singleton(messageDescriptor.toString())); + return acceptCount(scanResultBuilder, severity, 1) + .build(); } /** diff --git a/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy b/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy index 91bd09a..20adfc2 100644 --- a/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy +++ b/exec/src/test/groovy/com/optum/sourcehawk/exec/ExecOptionsSpec.groovy @@ -4,6 +4,7 @@ import com.optum.sourcehawk.core.data.RemoteRef import com.optum.sourcehawk.core.repository.LocalRepositoryFileReader import com.optum.sourcehawk.core.data.Verbosity +import com.optum.sourcehawk.core.repository.RemoteRepositoryFileReader import spock.lang.Specification import java.nio.file.Paths @@ -54,15 +55,15 @@ class ExecOptionsSpec extends Specification { !execOptions.remoteRef } - def "builder - github"() { + def "builder - remote"() { given: - RemoteRef remoteRef = RemoteRef.parse(RemoteRef.Type.GITHUB, "owner/repo@main") + RemoteRef remoteRef = RemoteRef.parse("owner/repo", "main") ExecOptions.ExecOptionsBuilder builder = ExecOptions.builder() .repositoryRoot(Paths.get("/")) .configurationFileLocation("Sourcehawk") .verbosity(Verbosity.ZERO) .failOnWarnings(true) - .repositoryFileReader(new GithubRepositoryFileReader("token", remoteRef)) + .repositoryFileReader(new RemoteRepositoryFileReader("https://raw.githubusercontent.com/owner/repo/main/%s", Collections.emptyMap())) .remoteRef(remoteRef) when: @@ -75,7 +76,7 @@ class ExecOptionsSpec extends Specification { execOptions.verbosity == Verbosity.ZERO !execOptions.tags execOptions.failOnWarnings - execOptions.repositoryFileReader instanceof GithubRepositoryFileReader + execOptions.repositoryFileReader instanceof RemoteRepositoryFileReader execOptions.remoteRef == remoteRef } diff --git a/exec/src/test/resources/sourcehawk-flattened-base.yml b/exec/src/test/resources/sourcehawk-flattened-base.yml index b250224..efcca44 100644 --- a/exec/src/test/resources/sourcehawk-flattened-base.yml +++ b/exec/src/test/resources/sourcehawk-flattened-base.yml @@ -53,4 +53,4 @@ file-protocols: enforcers: - enforcer: ".common.StringPropertyEquals" property-name: "distributionUrl" - expected-property-value: "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip" + expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip" diff --git a/pom.xml b/pom.xml index 8ecf522..c19b514 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ sourcehawk - 0.6.0-SNAPSHOT + 0.6.1-SNAPSHOT pom Sourcehawk From 74635f5b7d78d8901f422a329bfa47e4d4322649 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 14:43:31 -0500 Subject: [PATCH 03/35] remote-repository: Fix maven version --- .mvn/wrapper/maven-wrapper.properties | 2 +- cli/src/test/resources/flattened/sourcehawk-flattened-base.yml | 2 +- exec/src/test/resources/sourcehawk-flattened-base.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 26ae971..f8b5614 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -1 +1 @@ -distributionUrl = https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip \ No newline at end of file +distributionUrl = https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip \ No newline at end of file diff --git a/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml b/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml index efcca44..2a183f9 100644 --- a/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml +++ b/cli/src/test/resources/flattened/sourcehawk-flattened-base.yml @@ -53,4 +53,4 @@ file-protocols: enforcers: - enforcer: ".common.StringPropertyEquals" property-name: "distributionUrl" - expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip" + expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip" diff --git a/exec/src/test/resources/sourcehawk-flattened-base.yml b/exec/src/test/resources/sourcehawk-flattened-base.yml index efcca44..2a183f9 100644 --- a/exec/src/test/resources/sourcehawk-flattened-base.yml +++ b/exec/src/test/resources/sourcehawk-flattened-base.yml @@ -53,4 +53,4 @@ file-protocols: enforcers: - enforcer: ".common.StringPropertyEquals" property-name: "distributionUrl" - expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip" + expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip" From ac60c9321010c71c67735206f305255566a3ceec Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 18:47:36 -0500 Subject: [PATCH 04/35] remote-repository: Update groovy to 3.0.9 for JDK 17 support --- attribution.txt | 4 ++-- pom.xml | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/attribution.txt b/attribution.txt index e9b35ea..e3879aa 100644 --- a/attribution.txt +++ b/attribution.txt @@ -390,7 +390,7 @@ https://opensource.org/licenses/BSD-2-Clause ------------------------------------------------------------------------------------------------------------------------------- -Package: org.spockframework:spock-core:2.0-M3-groovy-3.0 +Package: org.spockframework:spock-core:2.0-groovy-3.0 License: Apache-2.0 @@ -1847,7 +1847,7 @@ limitations under the License. ------------------------------------------------------------------------------------------------------------------------------- -Package: org.codehaus.groovy:groovy:3.0.4 +Package: org.codehaus.groovy:groovy:3.0.9 License: Apache-2.0 diff --git a/pom.xml b/pom.xml index c19b514..de5374f 100644 --- a/pom.xml +++ b/pom.xml @@ -108,7 +108,13 @@ org.spockframework spock-core - 2.0-M3-groovy-3.0 + 2.0-groovy-3.0 + test + + + org.codehaus.groovy + groovy + ${groovy.version} test From 546c03a7a120d38833b21514ec7f47429e27e071 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 18:55:04 -0500 Subject: [PATCH 05/35] remote-repository: Fix maven version in test --- .../resources/flattened/sourcehawk-flattened-override.yml | 2 +- pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml b/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml index 8c715e9..865aeb5 100644 --- a/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml +++ b/cli/src/test/resources/flattened/sourcehawk-flattened-override.yml @@ -72,4 +72,4 @@ file-protocols: enforcers: - enforcer: ".common.StringPropertyEquals" property-name: "distributionUrl" - expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip" + expected-property-value: "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip" diff --git a/pom.xml b/pom.xml index de5374f..ab8b605 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 0.97 - 1.7.30 + 1.7.32 e1 @@ -120,7 +120,7 @@ junit junit - 4.13.1 + 4.13.2 test From d80fc0539bf21c3971a2e687e7fce97bbb9391b9 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 19:17:46 -0500 Subject: [PATCH 06/35] remote-repository: Fix more tests for JDK 17 --- .../com/optum/sourcehawk/cli/CliBaseSpecification.groovy | 2 +- .../aot/SourcehawkFileEnforcerRegistryProcessorSpec.groovy | 4 +++- .../com/optum/sourcehawk/exec/ConfigurationReaderSpec.groovy | 5 ++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/CliBaseSpecification.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/CliBaseSpecification.groovy index fafc98c..690f599 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/CliBaseSpecification.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/CliBaseSpecification.groovy @@ -31,7 +31,7 @@ class CliBaseSpecification extends Specification { } def cleanupSpec() { - System.setSecurityManager(defaultSecurityManager) + System.setSecurityManager(defaultSecurityManager) // TODO: Deprecated starting in JDK 17 } protected void createParentDirectories(final File child) { diff --git a/enforcer/file/aot/src/test/groovy/com/optum/sourcehawk/enforcer/file/aot/SourcehawkFileEnforcerRegistryProcessorSpec.groovy b/enforcer/file/aot/src/test/groovy/com/optum/sourcehawk/enforcer/file/aot/SourcehawkFileEnforcerRegistryProcessorSpec.groovy index 1aa5c6f..d657afd 100644 --- a/enforcer/file/aot/src/test/groovy/com/optum/sourcehawk/enforcer/file/aot/SourcehawkFileEnforcerRegistryProcessorSpec.groovy +++ b/enforcer/file/aot/src/test/groovy/com/optum/sourcehawk/enforcer/file/aot/SourcehawkFileEnforcerRegistryProcessorSpec.groovy @@ -154,7 +154,9 @@ class SourcehawkFileEnforcerRegistryProcessorSpec extends Specification { 1 * mockProcessingEnvironment.getFiler() >> mockFiler 1 * mockFiler.createSourceFile("com.optum.sourcehawk.enforcer.file.FileEnforcerRegistry", []) >> null 1 * mockProcessingEnvironment.getMessager() >> mockMessager - 1 * mockMessager.printMessage(Diagnostic.Kind.ERROR, 'Unable to generate file enforcer registry: java.lang.NullPointerException') + 1 * mockMessager.printMessage(Diagnostic.Kind.ERROR, _ as String) >> { kind, msg -> + msg == 'Unable to generate file enforcer registry: java.lang.NullPointerException' + } 0 * _ and: diff --git a/exec/src/test/groovy/com/optum/sourcehawk/exec/ConfigurationReaderSpec.groovy b/exec/src/test/groovy/com/optum/sourcehawk/exec/ConfigurationReaderSpec.groovy index d4ee1fe..ba6c9cf 100644 --- a/exec/src/test/groovy/com/optum/sourcehawk/exec/ConfigurationReaderSpec.groovy +++ b/exec/src/test/groovy/com/optum/sourcehawk/exec/ConfigurationReaderSpec.groovy @@ -2,7 +2,6 @@ package com.optum.sourcehawk.exec import com.optum.sourcehawk.core.configuration.SourcehawkConfiguration import org.spockframework.util.IoUtil -import sun.nio.ch.ChannelInputStream class ConfigurationReaderSpec extends FileBaseSpecification { @@ -38,7 +37,7 @@ class ConfigurationReaderSpec extends FileBaseSpecification { InputStream inputStream = ConfigurationReader.obtainInputStream(repositoryRoot, configurationFileLocation).get() then: - inputStream instanceof ChannelInputStream + inputStream.class.name == 'sun.nio.ch.ChannelInputStream' } def "obtainInputStream - relative file"() { @@ -50,7 +49,7 @@ class ConfigurationReaderSpec extends FileBaseSpecification { then: inputStream - inputStream instanceof ChannelInputStream + inputStream.class.name == 'sun.nio.ch.ChannelInputStream' } def "merge"() { From 2068edaf2979d1555518ae8ad0cb61ae14bc4d7a Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 19:27:25 -0500 Subject: [PATCH 07/35] remote-repository: Fix Dockerfile ci images references --- distributions/linux/native-image-builder/Dockerfile | 2 +- distributions/rpm/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distributions/linux/native-image-builder/Dockerfile b/distributions/linux/native-image-builder/Dockerfile index ce6898d..c2cda94 100644 --- a/distributions/linux/native-image-builder/Dockerfile +++ b/distributions/linux/native-image-builder/Dockerfile @@ -1,5 +1,5 @@ ARG GRAALVM_VERSION=21.3.0-java8 -FROM ghcr.io/optum/sourcehawk-ci/nativeimage:graalvm-ce-${GRAALVM_VERSION} +FROM ghcr.io/optum/ci/nativeimage:graalvm-ce-${GRAALVM_VERSION} # Build Arguments ARG NAME diff --git a/distributions/rpm/Dockerfile b/distributions/rpm/Dockerfile index 7b8d31a..430d963 100644 --- a/distributions/rpm/Dockerfile +++ b/distributions/rpm/Dockerfile @@ -1,5 +1,5 @@ ARG FROM_TAG=centos7 -FROM ghcr.io/optum/sourcehawk-ci/rpmbuild:${FROM_TAG} +FROM ghcr.io/optum/ci/rpmbuild:${FROM_TAG} # Build Arguments ARG RPM_BUILD_DIRECTORY From cc2fa222dd433be73948e274cf63f3ff15adc1b3 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 19:32:10 -0500 Subject: [PATCH 08/35] remote-repository: Fix java8 graalvm version --- distributions/linux/native-image-builder/Dockerfile | 2 +- distributions/linux/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distributions/linux/native-image-builder/Dockerfile b/distributions/linux/native-image-builder/Dockerfile index c2cda94..2551249 100644 --- a/distributions/linux/native-image-builder/Dockerfile +++ b/distributions/linux/native-image-builder/Dockerfile @@ -1,4 +1,4 @@ -ARG GRAALVM_VERSION=21.3.0-java8 +ARG GRAALVM_VERSION=21.2.0-java8 FROM ghcr.io/optum/ci/nativeimage:graalvm-ce-${GRAALVM_VERSION} # Build Arguments diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index be5e6dc..610b8ea 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -236,7 +236,7 @@ 8 - 21.3.0-java8 + 21.2.0-java8 From 7ddac548fcd4fed923e94340cc54490e2d29edc8 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 20:09:27 -0500 Subject: [PATCH 09/35] remote-repository: Vulnerability fixes, Fixes #38, Fixes #59, update ci with docker login and graalvm github actions --- .github/workflows/maven-ci.yml | 35 ++++++++++++++++++---------------- .github/workflows/release.yml | 35 ++++++++++++++++++---------------- attribution.txt | 2 +- enforcer/file/maven/pom.xml | 2 +- 4 files changed, 40 insertions(+), 34 deletions(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index aae2c26..c70b4f6 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -61,6 +61,12 @@ jobs: server-password: SONATYPE_PASSWORD gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} gpg-passphrase: SONATYPE_GPG_PASSPHRASE + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set Maven Project Version shell: bash run: | @@ -138,11 +144,12 @@ jobs: with: java-version: 8 - name: Setup GraalVM - uses: DeLaGuardo/setup-graalvm@master + uses: graalvm/setup-graalvm@v1 with: - graalvm-version: 21.2.0.java8 - - name: Setup GraalVM Native Image Tool - run: gu install native-image + version: '21.2.0' + java-version: '8' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Mac Native Image if: success() working-directory: build @@ -180,20 +187,16 @@ jobs: - name: Rename Native Image JAR working-directory: build run: ren *.jar native-image.jar - - name: Setup GraalVM Native Image and Visual C Build Tools - run: | - Invoke-RestMethod -Uri https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.3.0/graalvm-ce-java11-windows-amd64-21.3.0.zip -OutFile 'graal.zip' - Expand-Archive -path 'graal.zip' -destinationpath '.' - graalvm-ce-java11-21.3.0\bin\gu.cmd install native-image - choco install visualstudio2017-workload-vctools + - name: Setup GraalVM + uses: graalvm/setup-graalvm@v1 + with: + version: '21.3.0' + java-version: '11' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Windows Native Image if: success() - shell: cmd - run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat" - graalvm-ce-java11-21.3.0\bin\native-image -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime - env: - JAVA_HOME: ./graalvm-ce-java11-21.3.0 + run: native-image.cmd -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime - name: Archive Windows Native Image if: success() continue-on-error: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 519f563..2cea482 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,6 +37,12 @@ jobs: server-password: SONATYPE_PASSWORD gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} gpg-passphrase: SONATYPE_GPG_PASSPHRASE + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set Maven Project Version id: set_maven_project_version shell: bash @@ -226,11 +232,12 @@ jobs: with: java-version: 8 - name: Setup GraalVM - uses: DeLaGuardo/setup-graalvm@master + uses: graalvm/setup-graalvm@v1 with: - graalvm-version: 21.2.0.java8 - - name: Setup GraalVM Native Image Tool - run: gu install native-image + version: '21.2.0' + java-version: '8' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Mac Native Image if: success() working-directory: build @@ -289,20 +296,16 @@ jobs: - name: Rename Native Image JAR working-directory: build run: ren *.jar native-image.jar - - name: Setup GraalVM Native Image and Visual C Build Tools - run: | - Invoke-RestMethod -Uri https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.3.0/graalvm-ce-java11-windows-amd64-21.3.0.zip -OutFile 'graal.zip' - Expand-Archive -path 'graal.zip' -destinationpath '.' - graalvm-ce-java11-21.3.0\bin\gu.cmd install native-image - choco install visualstudio2017-workload-vctools + - name: Setup GraalVM + uses: graalvm/setup-graalvm@v1 + with: + version: '21.3.0' + java-version: '11' + components: 'native-image' + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Windows Native Image if: success() - shell: cmd - run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat" - graalvm-ce-java11-21.3.0\bin\native-image -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime - env: - JAVA_HOME: ./graalvm-ce-java11-21.3.0 + run: native-image.cmd -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime - name: Smoke Test if: success() shell: cmd diff --git a/attribution.txt b/attribution.txt index e3879aa..75d4a50 100644 --- a/attribution.txt +++ b/attribution.txt @@ -1955,7 +1955,7 @@ limitations under the License. ------------------------------------------------------------------------------------------------------------------------------- -Package: org.apache.maven:maven-model:3.6.3 +Package: org.apache.maven:maven-model:3.8.4 License: Apache-2.0 diff --git a/enforcer/file/maven/pom.xml b/enforcer/file/maven/pom.xml index fba5aea..699b4cf 100644 --- a/enforcer/file/maven/pom.xml +++ b/enforcer/file/maven/pom.xml @@ -27,7 +27,7 @@ org.apache.maven maven-model - 3.6.3 + ${maven.version} compile From 915366dab64f8a0ee05b519791f2312964c2621c Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 20:13:46 -0500 Subject: [PATCH 10/35] remote-repository: Login to GHCR with correct creds --- .github/workflows/maven-ci.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index c70b4f6..423c1b0 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -65,8 +65,8 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.GIT_USERNAME }} + password: ${{ secrets.GIT_PACKAGE_TOKEN }} - name: Set Maven Project Version shell: bash run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2cea482..bf4aeb7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,8 +41,8 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.GIT_USERNAME }} + password: ${{ secrets.GIT_PACKAGE_TOKEN }} - name: Set Maven Project Version id: set_maven_project_version shell: bash From 31c48bf5040b7acccc9a5140ec56c4b184d16a41 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 20:39:42 -0500 Subject: [PATCH 11/35] remote-repository: Update static native imate to be -H:+StaticExecutableWithDynamicLibC --- .../native-image/sourcehawk-cli/native-image.properties | 3 +-- .../com/optum/sourcehawk/cli/NativeImageConfigSpec.groovy | 1 - distributions/linux/native-image-builder/Dockerfile | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cli/src/main/resources/META-INF/native-image/sourcehawk-cli/native-image.properties b/cli/src/main/resources/META-INF/native-image/sourcehawk-cli/native-image.properties index 10ea548..57ad3db 100644 --- a/cli/src/main/resources/META-INF/native-image/sourcehawk-cli/native-image.properties +++ b/cli/src/main/resources/META-INF/native-image/sourcehawk-cli/native-image.properties @@ -1,3 +1,2 @@ Args = -H:Class=${cli.class} \ - -H:Name=${cli.name} \ - --no-server \ No newline at end of file + -H:Name=${cli.name} \ No newline at end of file diff --git a/cli/src/test/groovy/com/optum/sourcehawk/cli/NativeImageConfigSpec.groovy b/cli/src/test/groovy/com/optum/sourcehawk/cli/NativeImageConfigSpec.groovy index e657673..85ad9bc 100644 --- a/cli/src/test/groovy/com/optum/sourcehawk/cli/NativeImageConfigSpec.groovy +++ b/cli/src/test/groovy/com/optum/sourcehawk/cli/NativeImageConfigSpec.groovy @@ -19,7 +19,6 @@ class NativeImageConfigSpec extends Specification { then: args.contains("-H:Class=${Sourcehawk.name}") args.contains("-H:Name=sourcehawk") - args.contains("--no-server") } def "all native image configs are on classpath"() { diff --git a/distributions/linux/native-image-builder/Dockerfile b/distributions/linux/native-image-builder/Dockerfile index 2551249..23d2770 100644 --- a/distributions/linux/native-image-builder/Dockerfile +++ b/distributions/linux/native-image-builder/Dockerfile @@ -15,6 +15,6 @@ WORKDIR ${WORKDIR} # Build the native image RUN native-image -cp native-image.jar \ -H:+ReportExceptionStackTraces \ + -H:+StaticExecutableWithDynamicLibC \ --report-unsupported-elements-at-runtime \ - --no-fallback \ - --static \ No newline at end of file + --no-fallback \ No newline at end of file From 0dbbc017013b3d15bed3be65f55e4d033527b0c1 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Thu, 6 Jan 2022 21:03:58 -0500 Subject: [PATCH 12/35] remote-repository: Update busybox --- distributions/docker/Dockerfile | 2 +- distributions/linux/native-image-builder/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distributions/docker/Dockerfile b/distributions/docker/Dockerfile index 8bb6827..5b3d26b 100644 --- a/distributions/docker/Dockerfile +++ b/distributions/docker/Dockerfile @@ -1,5 +1,5 @@ # glibc is required for proper DNS resolution within app -FROM busybox:1.32.0-glibc +FROM busybox:1.35.0-glibc # Dynamically pass in name ARG NAME="sourcehawk" diff --git a/distributions/linux/native-image-builder/Dockerfile b/distributions/linux/native-image-builder/Dockerfile index 23d2770..2551249 100644 --- a/distributions/linux/native-image-builder/Dockerfile +++ b/distributions/linux/native-image-builder/Dockerfile @@ -15,6 +15,6 @@ WORKDIR ${WORKDIR} # Build the native image RUN native-image -cp native-image.jar \ -H:+ReportExceptionStackTraces \ - -H:+StaticExecutableWithDynamicLibC \ --report-unsupported-elements-at-runtime \ - --no-fallback \ No newline at end of file + --no-fallback \ + --static \ No newline at end of file From fe26db44f6ed86cd05ce22f481102c8a911ce4be Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 16:07:00 -0500 Subject: [PATCH 13/35] remote-repository: Static native image builds on musl, scratch Docker image, disabling Bintray publish --- .github/workflows/maven-ci.yml | 2 +- .github/workflows/release.yml | 33 ++ .gitignore | 1 + NOTICE.txt | 2 +- attribution.txt | 2 +- cli/pom.xml | 2 +- cli/scripts/update-picocli.sh | 22 +- .../cli/AbstractRemoteScanCommand.java | 2 +- .../com/optum/sourcehawk/cli/Sourcehawk.java | 2 +- cli/src/main/java/picocli/CommandLine.java | 168 ++++++--- distributions/debian/pom.xml | 181 +++++----- .../docker-builders/Dockerfile-nativeimage | 15 +- distributions/docker/Dockerfile | 22 +- distributions/docker/README.md | 6 +- distributions/docker/etc/group | 1 + distributions/docker/etc/passwd | 1 + distributions/docker/pom.xml | 30 +- .../linux/native-image-builder/Dockerfile | 3 +- distributions/linux/pom.xml | 56 ++- distributions/rpm/pom.xml | 325 ++++++++---------- .../scripts/build-and-push-docker-builders.sh | 4 - 21 files changed, 496 insertions(+), 384 deletions(-) create mode 100644 distributions/docker/etc/group create mode 100644 distributions/docker/etc/passwd diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index 423c1b0..5ff5f60 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -126,7 +126,7 @@ jobs: steps: - uses: actions/download-artifact@v2 with: - name: native-image-8 + name: native-image-11 path: build - uses: actions/download-artifact@v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bf4aeb7..bfa76b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -149,6 +149,17 @@ jobs: asset_path: ./distributions/debian/target/sourcehawk-ubuntu-focal.deb asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-ubuntu-focal-amd64.deb asset_content_type: application/octet-stream + - name: Upload Sourcehawk Centos 7 RPM Package + if: success() + continue-on-error: true + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./distributions/rpm/target/sourcehawk-centos-7.rpm + asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.el7.x86_64.rpm + asset_content_type: application/octet-stream - name: Upload Sourcehawk Centos 8 RPM Package if: success() continue-on-error: true @@ -160,6 +171,28 @@ jobs: asset_path: ./distributions/rpm/target/sourcehawk-centos-8.rpm asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.el8.x86_64.rpm asset_content_type: application/octet-stream + - name: Upload Sourcehawk Fedora 33 RPM Package + if: success() + continue-on-error: true + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./distributions/rpm/target/sourcehawk-fedora-33.rpm + asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.fc33.x86_64.rpm + asset_content_type: application/octet-stream + - name: Upload Sourcehawk Fedora 34 RPM Package + if: success() + continue-on-error: true + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./distributions/rpm/target/sourcehawk-fedora-34.rpm + asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.fc34.x86_64.rpm + asset_content_type: application/octet-stream - name: Upload Sourcehawk Fedora 35 RPM Package if: success() continue-on-error: true diff --git a/.gitignore b/.gitignore index 0e332c5..6028a98 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ target/ *.log log logs +*.bak ### STS ### .apt_generated diff --git a/NOTICE.txt b/NOTICE.txt index 15afa52..4fc5b89 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,6 +1,6 @@ sourcehawk -Copyright 2021 Optum +Copyright 2022 Optum Project Description: ==================== diff --git a/attribution.txt b/attribution.txt index 75d4a50..fc3dc30 100644 --- a/attribution.txt +++ b/attribution.txt @@ -2230,7 +2230,7 @@ http://www.apache.org/licenses/LICENSE-2.0 ------------------------------------------------------------------------------------------------------------------------------- -Package: info.picocli:4.6.1 +Package: info.picocli:4.6.2 License: Apache-2.0 diff --git a/cli/pom.xml b/cli/pom.xml index aeb3dee..624debf 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -31,7 +31,7 @@ **/picocli/**/*.* - 4.6.1 + 4.6.2 diff --git a/cli/scripts/update-picocli.sh b/cli/scripts/update-picocli.sh index f4619ee..3d35f7d 100755 --- a/cli/scripts/update-picocli.sh +++ b/cli/scripts/update-picocli.sh @@ -11,11 +11,11 @@ set -e ######################################################################### # Retrieve Latest Version -VERSION=$(curl -sI https://github.com/remkop/picocli/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d 'v' | tr -d '\r\n') +VERSION=$(curl -ksI https://github.com/remkop/picocli/releases/latest | grep -i location: | awk -F"/" '{ printf "%s", $NF }' | tr -d 'v' | tr -d '\r\n') # Global Variables -DIR="$( cd "$( dirname "$( dirname "${BASH_SOURCE[0]}" )")" && pwd )" -ROOT_DIR="$( cd "$( dirname "$( dirname "$( dirname "${BASH_SOURCE[0]}" )")")" && pwd )" +DIR="$(dirname "$(cd -- "$(dirname "$0")"; pwd -P)")" +ROOT_DIR="$(dirname "$(dirname "$(cd -- "$(dirname "$0")"; pwd -P)")")" BASE_URL="https://raw.githubusercontent.com/remkop/picocli" LICENSE_URL="$BASE_URL/v$VERSION/LICENSE" LICENSE_FILE_PATH="$DIR/src/main/resources/META-INF/licenses/picocli.txt" @@ -28,15 +28,19 @@ curl -ksf "$LICENSE_URL" > "$LICENSE_FILE_PATH" curl -ksf "$SOURCE_URL" > "$SOURCE_FILE_PATH" # Add some warning suppression to the java source file -sed -i 's/public\sclass\sCommandLine/@SuppressWarnings({"rawtypes", "deprecation" })\npublic class CommandLine/g' "$SOURCE_FILE_PATH" +sed -i.bak -e 's/public\sclass\sCommandLine/@SuppressWarnings({"rawtypes", "deprecation" })\npublic class CommandLine/g' \ + -e 's/TODO/TIDO/g' "$SOURCE_FILE_PATH" \ + && rm -rf "$SOURCE_FILE_PATH.bak" + +# Remove TODOs so not highlighted in editor +sed -i.bak 's/TODO/TIDO/g' "$SOURCE_FILE_PATH" # Replace the version in pom.xml file for plugin references -sed -i "s/[-[:alnum:]./]\{1,\}<\/picocli.version>/$VERSION<\/picocli.version>/" "$DIR/pom.xml" +sed -i.bak "s/[-[:alnum:]./]\{1,\}<\/picocli.version>/$VERSION<\/picocli.version>/" "$DIR/pom.xml" \ + && rm -rf "$DIR/pom.xml.bak" # Replace the version in attribution.txt file -sed -i "s/Package: info.picocli:[-[:alnum:]./]\{1,\}/Package: info.picocli:$VERSION/" "$ROOT_DIR/attribution.txt" - -# Remove TODOs so not highlighted in editor -sed -i 's/TODO/TIDO/g' "$SOURCE_FILE_PATH" +sed -i.bak "s/Package: info.picocli:[-[:alnum:]./]\{1,\}/Package: info.picocli:$VERSION/" "$ROOT_DIR/attribution.txt" \ + && rm -rf "$ROOT_DIR/attribution.txt.bak" echo "Picocli updated to version: $VERSION" \ No newline at end of file diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java b/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java index cdae269..fffff82 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java @@ -43,7 +43,7 @@ abstract class AbstractRemoteScanCommand implements Callable { */ @Override public Integer call() { - val parentExecOptions = parentCommand.buildExecOptions(); + val parentExecOptions = parentCommand.buildExecOptions(); // TODO: NPE ?? val execOptionsBuilder = parentExecOptions.toBuilder(); val configFileProvided = Optional.ofNullable(parentCommand.spec) .map(CommandLine.Model.CommandSpec::commandLine) diff --git a/cli/src/main/java/com/optum/sourcehawk/cli/Sourcehawk.java b/cli/src/main/java/com/optum/sourcehawk/cli/Sourcehawk.java index 28f1e86..8840dab 100644 --- a/cli/src/main/java/com/optum/sourcehawk/cli/Sourcehawk.java +++ b/cli/src/main/java/com/optum/sourcehawk/cli/Sourcehawk.java @@ -22,7 +22,7 @@ headerHeading = "@|fg(magenta) >_ S O U R C E H A W K|@", synopsisHeading = "%n", commandListHeading ="%nCommands:%n", - footer = "Copyright (c) 2020 Optum", + footer = "Copyright (c) 2022 Optum", versionProvider = Sourcehawk.VersionProvider.class, subcommands = { CommandLine.HelpCommand.class, diff --git a/cli/src/main/java/picocli/CommandLine.java b/cli/src/main/java/picocli/CommandLine.java index 636662b..a0f8454 100644 --- a/cli/src/main/java/picocli/CommandLine.java +++ b/cli/src/main/java/picocli/CommandLine.java @@ -72,7 +72,7 @@ * * // CheckSum implements Callable, so parsing, error handling and handling user * // requests for usage help or version help can be done with one line of code. - * public static void main(String[] args) throws Exception { + * public static void main(String[] args) { * int exitCode = new CommandLine(new CheckSum()).execute(args); * System.exit(exitCode); * } @@ -142,11 +142,10 @@ * Classes Related to Parsing Command Line Arguments *

*/ -@SuppressWarnings({"rawtypes", "deprecation" }) public class CommandLine { /** This is picocli version {@value}. */ - public static final String VERSION = "4.6.1"; + public static final String VERSION = "4.6.2"; private final Tracer tracer = new Tracer(); private CommandSpec commandSpec; @@ -1207,7 +1206,7 @@ public CommandLine setColorScheme(Help.ColorScheme colorScheme) { * help with a {@code --help} or similar option, the usage help message is printed to the standard output stream so that it can be easily searched and paged.

* @since 4.0 */ public PrintWriter getOut() { - if (out == null) { setOut(new PrintWriter(System.out, true)); } + if (out == null) { setOut(newPrintWriter(System.out, getStdoutEncoding())); } return out; } @@ -1234,7 +1233,7 @@ public CommandLine setOut(PrintWriter out) { * should use this writer to print error messages (which may include a usage help message) when an unexpected error occurs.

* @since 4.0 */ public PrintWriter getErr() { - if (err == null) { setErr(new PrintWriter(System.err, true)); } + if (err == null) { setErr(newPrintWriter(System.err, getStderrEncoding())); } return err; } @@ -1808,7 +1807,7 @@ protected R throwOrExit(ExecutionException ex) { * @since 2.0 */ @Deprecated public static class DefaultExceptionHandler extends AbstractHandler> implements IExceptionHandler, IExceptionHandler2 { public List handleException(ParameterException ex, PrintStream out, Help.Ansi ansi, String... args) { - internalHandleParseException(ex, new PrintWriter(out, true), Help.defaultColorScheme(ansi)); return Collections.emptyList(); } + internalHandleParseException(ex, newPrintWriter(out, getStdoutEncoding()), Help.defaultColorScheme(ansi)); return Collections.emptyList(); } /** Prints the message of the specified exception, followed by the usage message for the command or subcommand * whose input was invalid, to the stream returned by {@link #err()}. @@ -1818,7 +1817,7 @@ public List handleException(ParameterException ex, PrintStream out, Help * @return the empty list * @since 3.0 */ public R handleParseException(ParameterException ex, String[] args) { - internalHandleParseException(ex, new PrintWriter(err(), true), colorScheme()); return returnResultOrExit(null); } + internalHandleParseException(ex, newPrintWriter(err(), getStderrEncoding()), colorScheme()); return returnResultOrExit(null); } static void internalHandleParseException(ParameterException ex, PrintWriter writer, Help.ColorScheme colorScheme) { writer.println(colorScheme.errorText(ex.getMessage())); @@ -1879,7 +1878,7 @@ public static boolean printHelpIfRequested(ParseResult parseResult) { * @since 3.6 */ @Deprecated public static boolean printHelpIfRequested(List parsedCommands, PrintStream out, PrintStream err, Help.ColorScheme colorScheme) { // for backwards compatibility - for (CommandLine cmd : parsedCommands) { cmd.setOut(new PrintWriter(out, true)).setErr(new PrintWriter(err, true)).setColorScheme(colorScheme); } + for (CommandLine cmd : parsedCommands) { cmd.setOut(newPrintWriter(out, getStdoutEncoding())).setErr(newPrintWriter(err, getStderrEncoding())).setColorScheme(colorScheme); } return executeHelpRequest(parsedCommands) != null; } @@ -2135,8 +2134,8 @@ private T enrichForBackwardsCompatibility(T obj) { // and the application called #useOut, #useErr or #useAnsi on it if (obj instanceof AbstractHandler) { AbstractHandler handler = (AbstractHandler) obj; - if (handler.out() != System.out) { setOut(new PrintWriter(handler.out(), true)); } - if (handler.err() != System.err) { setErr(new PrintWriter(handler.err(), true)); } + if (handler.out() != System.out) { setOut(newPrintWriter(handler.out(), getStdoutEncoding())); } + if (handler.err() != System.err) { setErr(newPrintWriter(handler.err(), getStderrEncoding())); } if (handler.ansi() != Help.Ansi.AUTO) { setColorScheme(handler.colorScheme()); } } return obj; @@ -2229,6 +2228,9 @@ private int resolveExitCode(int exitCodeOnSuccess, R executionResult, List> implements IParseResultHandler { + /** {@inheritDoc} */ + public int execute(ParseResult parseResult) throws ExecutionException { return super.execute(parseResult); } + /** Prints help if requested, and otherwise executes the top-level {@code Runnable} or {@code Callable} command. * Finally, either a list of result objects is returned, or the JVM is terminated if an exit code {@linkplain #andExit(int) was set}. * If the top-level command does not implement either {@code Runnable} or {@code Callable}, an {@code ExecutionException} @@ -2310,6 +2312,9 @@ protected List extractExitCodeGenerators(ParseResult parseRe *

* @since 2.0 */ public static class RunLast extends AbstractParseResultHandler> implements IParseResultHandler { + /** {@inheritDoc} */ + public int execute(ParseResult parseResult) throws ExecutionException { return super.execute(parseResult); } + /** Prints help if requested, and otherwise executes the most specific {@code Runnable} or {@code Callable} subcommand. *

For {@linkplain Command#subcommandsRepeatable() repeatable subcommands}, this method * may execute multiple subcommands: the most deeply nested subcommands that have the same parent command.

@@ -2382,6 +2387,9 @@ protected List extractExitCodeGenerators(ParseResult parseRe * For use by the {@link #execute(String...) execute} method. * @since 2.0 */ public static class RunAll extends AbstractParseResultHandler> implements IParseResultHandler { + /** {@inheritDoc} */ + public int execute(ParseResult parseResult) throws ExecutionException { return super.execute(parseResult); } + /** Prints help if requested, and otherwise executes the top-level command and all subcommands as {@code Runnable}, * {@code Callable} or {@code Method}. Finally, either a list of result objects is returned, or the JVM is terminated if an exit * code {@linkplain #andExit(int) was set}. If any of the {@code CommandLine} commands does not implement either @@ -3648,22 +3656,33 @@ public enum ScopeType { boolean required() default false; /** - * Set {@code help=true} if this option should disable validation of the remaining arguments: - * If the {@code help} option is specified, no error message is generated for missing required options. - *

- * This attribute is useful for special options like help ({@code -h} and {@code --help} on unix, - * {@code -?} and {@code -Help} on Windows) or version ({@code -V} and {@code --version} on unix, - * {@code -Version} on Windows). + *

This should rarely be used: the recommended attributes are {@link #usageHelp() usageHelp} and {@link #versionHelp() versionHelp}. + *

+ * Only set {@code help=true} when this option should disable validation of the remaining + * arguments, and no error message should be generated for missing required options. + *

+ * This is useful for custom help options that are in addition to the standard help and + * version options. For example if your application has many hidden options or + * subcommands, and there is a custom help option like {@code --detailed-help} that prints + * the usage help message for these hidden options and subcommands. *

+ *

Note:

*

- * Note that the {@link #parse(String...)} method will not print help documentation. It will only set - * the value of the annotated field. It is the responsibility of the caller to inspect the annotated fields - * and take the appropriate action. + * Use the {@link #usageHelp() usageHelp} for "normal" help options (like {@code -h} and {@code --help} on unix, + * {@code -?} and {@code -Help} on Windows) + * and use {@link #versionHelp() versionHelp} for "normal" version help ({@code -V} and {@code --version} on unix, + * {@code -Version} on Windows): + * picocli has built-in logic so that options with {@code usageHelp=true} or {@code versionHelp=true} + * will automatically cause the requested help message to be printed in applications + * that use the {@link #execute(String...)} method, without any code in the application. + *

+ * Note that there is no such automatic help printing for options with {@code help=true}; + * applications need to check whether the end user specified this option and take appropriate action + * in the business logic of the application. *

* @return whether this option disables validation of the other arguments - * @deprecated Use {@link #usageHelp()} and {@link #versionHelp()} instead. See {@link #printHelpIfRequested(List, PrintStream, CommandLine.Help.Ansi)} */ - @Deprecated boolean help() default false; + boolean help() default false; /** * Set {@code usageHelp=true} for the {@code --help} option that triggers display of the usage help message. @@ -3735,7 +3754,7 @@ public enum ScopeType { * command line, a {@link MissingParameterException} is thrown by the {@link #parse(String...)} method. *

* In many cases picocli can deduce the number of required parameters from the field's type. - * By default, flags (boolean options) have arity zero, + * By default, flags (boolean options) have arity "0..1", * and single-valued type fields (String, int, Integer, double, Double, File, Date, etc) have arity one. * Generally, fields with types that cannot hold multiple values can omit the {@code arity} attribute. *

@@ -3747,7 +3766,8 @@ public enum ScopeType { *

* A note on boolean options *

- * By default picocli does not expect boolean options (also called "flags" or "switches") to have a parameter. + * By default picocli allows boolean options (also called "flags" or "switches") to have an optional parameter, + * which must be either "true" or "false" (lowercase, other values are rejected). * You can make a boolean option take a required parameter by annotating your field with {@code arity="1"}. * For example:

* 
@Option(names = "-v", arity = "1") boolean verbose;
@@ -3757,12 +3777,11 @@ public enum ScopeType { * on the command line, or a {@link MissingParameterException} is thrown by the {@link #parse(String...)} * method. *

- * To make the boolean parameter possible but optional, define the field with {@code arity = "0..1"}. + * To remove the optional parameter, define the field with {@code arity = "0"}. * For example:

- * 
@Option(names="-v", arity="0..1") boolean verbose;
- *

This will accept any of the below without throwing an exception:

+ * 
@Option(names="-v", arity="0") boolean verbose;
+ *

This will reject any of the below:

* 
-         * -v
          * -v true
          * -v false
          *
@@ -4974,6 +4993,8 @@ private static class NoDefaultProvider implements IDefaultValueProvider { * } * } * } + *

If this interface does not meet your requirements, you may have a look at the more powerful + * and flexible {@link IParameterPreprocessor} interface introduced with picocli 4.6.

* @see Option#parameterConsumer() * @see Parameters#parameterConsumer() * @since 4.0 */ @@ -5602,9 +5623,14 @@ static Range adjustForType(Range result, IAnnotatedElement member) { return result.isUnspecified ? defaultArity(member) : result; } /** Returns the default arity {@code Range}: for interactive options/positional parameters, - * this is 0; for {@link Option options} this is 0 for booleans and 1 for - * other types, for {@link Parameters parameters} booleans have arity 0, arrays or Collections have + * this is 0; for {@link Option options} this is effectively "0..1" for booleans and 1 for + * other types, for {@link Parameters parameters} booleans have arity 1, arrays or Collections have * arity "0..*", and other types have arity 1. + *

Implementation Notes

+ *

The returned {@code Range} for boolean options has an effective arity of "0..1". + * This is implemented by returning a {@code Range} with arity "0", + * and its {@code unspecified} property set to {@code true}. + * This implementation may change in the future.

* @param field the field whose default arity to return * @return a new {@code Range} indicating the default arity of the specified field * @since 2.0 */ @@ -6386,7 +6412,7 @@ public CommandSpec addSubcommand(String name, CommandSpec subcommand) { */ public CommandSpec addSubcommand(String name, CommandLine subCommandLine) { CommandSpec subSpec = subCommandLine.getCommandSpec(); - String actualName = validateSubcommandName(name, subSpec); + String actualName = validateSubcommandName(interpolator.interpolateCommandName(name), subSpec); Tracer t = new Tracer(); if (t.isDebug()) {t.debug("Adding subcommand '%s' to '%s'%n", actualName, this.qualifiedName());} String previousName = commands.getCaseSensitiveKey(actualName); @@ -6396,7 +6422,7 @@ public CommandSpec addSubcommand(String name, CommandLine subCommandLine) { subSpec.parent(this); for (String alias : subSpec.aliases()) { if (t.isDebug()) {t.debug("Adding alias '%s' for '%s'%n", (parent == null ? "" : parent.qualifiedName() + " ") + alias, this.qualifiedName());} - previous = commands.put(alias, subCommandLine); + previous = commands.put(interpolator.interpolate(alias), subCommandLine); if (previous != null && previous != subCommandLine) { throw new DuplicateNameException("Alias '" + alias + "' for subcommand '" + actualName + "' is already used by another subcommand of '" + this.name() + "'"); } } subSpec.initCommandHierarchyWithResourceBundle(resourceBundleBaseName(), resourceBundle()); @@ -6424,8 +6450,8 @@ private void inheritAttributesFrom(CommandSpec root) { updatedSubcommandsToInheritFrom(root); } private void updatedSubcommandsToInheritFrom(CommandSpec root) { - if (root != this) { - mixinStandardHelpOptions(root.mixinStandardHelpOptions()); + if (root != this && root.mixinStandardHelpOptions()) { // #1331 only add, don't remove + mixinStandardHelpOptions(true); } Set subcommands = new HashSet(subcommands().values()); for (CommandLine sub : subcommands) { @@ -6702,6 +6728,7 @@ public CommandSpec remove(ArgSpec arg) { if (positionalParameters.remove(arg)) { removed++; } + args.remove(arg); if (removed == 0) { throw new NoSuchElementException(String.valueOf(arg)); } @@ -6936,7 +6963,8 @@ public Set names() { public List args() { return Collections.unmodifiableList(args); } Object[] commandMethodParamValues() { Object[] values = new Object[methodParams.length]; - int argIndex = mixins.containsKey(AutoHelpMixin.KEY) ? 2 : 0; + CommandSpec autoHelpMixin = mixins.get(AutoHelpMixin.KEY); + int argIndex = autoHelpMixin == null || autoHelpMixin.inherited() ? 0 : 2; for (int i = 0; i < methodParams.length; i++) { if (methodParams[i].isAnnotationPresent(Mixin.class)) { String name = methodParams[i].getAnnotation(Mixin.class).name(); @@ -7155,7 +7183,20 @@ public CommandSpec negatableOptionTransformer(INegatableOptionTransformer newVal public CommandSpec mixinStandardHelpOptions(boolean newValue) { if (newValue) { CommandSpec mixin = CommandSpec.forAnnotatedObject(new AutoHelpMixin(), new DefaultFactory()); - addMixin(AutoHelpMixin.KEY, mixin); + boolean overlap = false; + for (String key : mixin.optionsMap().keySet()) { + if (optionsMap().containsKey(key)) { overlap = true; break; } + } + if (!overlap) { // #1316, 1319 avoid DuplicateOptionAnnotationsException + mixin.inherited = this.inherited(); + addMixin(AutoHelpMixin.KEY, mixin); + } + // #1331 if inherit(ed) we also add to subcommands + if (scopeType() == ScopeType.INHERIT || inherited()) { + for (CommandLine sub : new HashSet(subcommands().values())) { + sub.getCommandSpec().mixinStandardHelpOptions(newValue); + } + } } else { CommandSpec helpMixin = mixins.remove(AutoHelpMixin.KEY); if (helpMixin != null) { @@ -7167,11 +7208,7 @@ public CommandSpec mixinStandardHelpOptions(boolean newValue) { } } } - } - if (scopeType() == ScopeType.INHERIT || inherited()) { - for (CommandLine sub : new HashSet(subcommands().values())) { - sub.getCommandSpec().mixinStandardHelpOptions(newValue); - } + // #1331 we don't remove StandardHelpOptions from subcommands, even if they inherit from us } return this; } @@ -8669,10 +8706,20 @@ public String[] description() { private String[] expandVariables(String[] desc) { if (desc.length == 0) { return desc; } StringBuilder candidates = new StringBuilder(); - if (completionCandidates() != null) { - for (String c : completionCandidates()) { - if (candidates.length() > 0) { candidates.append(", "); } - candidates.append(c); + boolean isCompletionCandidatesUsed = false; + for (String s: desc) { + if (s.contains(DESCRIPTION_VARIABLE_COMPLETION_CANDIDATES)) { + isCompletionCandidatesUsed = true; + break; + } + } + if (isCompletionCandidatesUsed) { + Iterable iter = completionCandidates(); + if (iter != null) { + for (String c : iter) { + if (candidates.length() > 0) { candidates.append(", "); } + candidates.append(c); + } } } String defaultValueString = defaultValueString(false); // interpolate later @@ -8740,7 +8787,7 @@ private String[] expandVariables(String[] desc) { /** Returns the root option or positional parameter (on the parent command), if this option or positional parameter was inherited; * or {@code null} if it was not. * @see Option#scope() - * @since 4.6.1 */ + * @since 4.6.2 */ public ArgSpec root() { return root; } /** Returns the type to convert the option or positional parameter to before {@linkplain #setValue(Object) setting} the value. @@ -8787,8 +8834,10 @@ public String mapFallbackValue() { public Object initialValue() { // not not initialize if already CACHED, or UNAVAILABLE, or if annotatedElement==null if (initialValueState == InitialValueState.POSTPONED && annotatedElement != null) { - try { initialValue = annotatedElement.getter().get(); } catch (Exception ex) { } - initialValueState = InitialValueState.CACHED; + try { + initialValue = annotatedElement.getter().get(); + initialValueState = InitialValueState.CACHED; // only if successfully initialized + } catch (Exception ex) { } // #1300 if error: keep initialValueState == POSTPONED } return initialValue; } @@ -9376,7 +9425,7 @@ private static String inferLabel(String label, String fieldName, ITypeInfo typeI /** Returns the root option or positional parameter (on the parent command), if this option or positional parameter was inherited; * or {@code null} if it was not. * @see Option#scope() - * @since 4.6.1 */ + * @since 4.6.2 */ public ArgSpec root() { return root; } /** Returns the type to convert the option or positional parameter to before {@linkplain #setValue(Object) setting} the value. @@ -9519,7 +9568,7 @@ private static String inferLabel(String label, String fieldName, ITypeInfo typeI /** * Sets the root object for this inherited option, and returns this builder. - * @since 4.6.1 */ + * @since 4.6.2 */ public T root(ArgSpec root) { this.root = root ; return self(); } /** Sets the type to convert the option or positional parameter to before {@linkplain #setValue(Object) setting} the value, and returns this builder. @@ -13264,12 +13313,14 @@ private void processArguments(List parsedCommands, if (commandSpec.parent() != null && commandSpec.parent().subcommandsRepeatable() && commandSpec.parent().subcommands().containsKey(arg)) { tracer.debug("'%s' is a repeatable subcommand of %s%n", arg, commandSpec.parent().qualifiedName());// #454 repeatable subcommands CommandLine subcommand = commandSpec.parent().subcommands().get(arg); + Set inheritedInitialized = initialized; if (subcommand.interpreter.parseResultBuilder != null) { tracer.debug("Subcommand '%s' has been matched before. Making a copy...%n", subcommand.getCommandName()); subcommand = subcommand.copy(); subcommand.getCommandSpec().parent(commandSpec.parent()); // hook it up with its parent + inheritedInitialized = new LinkedHashSet(inheritedInitialized); } - processSubcommand(subcommand, getParent().interpreter.parseResultBuilder, parsedCommands, args, required, initialized, originalArgs, nowProcessing, separator, arg); + processSubcommand(subcommand, getParent().interpreter.parseResultBuilder, parsedCommands, args, required, inheritedInitialized, originalArgs, nowProcessing, separator, arg); continue; } @@ -14387,7 +14438,9 @@ private static String optionDescription(String prefix, ArgSpec argSpec, int opti if (argSpec.arity().max > 1) { desc += " at index " + optionParamIndex; } - desc += " (" + argSpec.paramLabel() + ")"; + if (argSpec.arity().max > 0) { + desc += " (" + argSpec.paramLabel() + ")"; + } } } else { desc = prefix + "positional parameter at index " + ((PositionalParamSpec) argSpec).index() + " (" + argSpec.paramLabel() + ")"; @@ -14469,6 +14522,17 @@ static void close(Closeable closeable) { new Tracer().warn("Could not close " + closeable + ": " + ex.toString()); } } + static Charset getStdoutEncoding() { + String encoding = System.getProperty("sun.stdout.encoding"); + return encoding != null ? Charset.forName(encoding) : Charset.defaultCharset(); + } + static Charset getStderrEncoding() { + String encoding = System.getProperty("sun.stderr.encoding"); + return encoding != null ? Charset.forName(encoding) : Charset.defaultCharset(); + } + static PrintWriter newPrintWriter(OutputStream stream, Charset charset) { + return new PrintWriter(new BufferedWriter(new OutputStreamWriter(stream, charset)), true); + } static class PositionalParametersSorter implements Comparator { private static final Range OPTION_INDEX = new Range(0, 0, false, true, "0"); public int compare(ArgSpec p1, ArgSpec p2) { @@ -18008,7 +18072,7 @@ static List stripErrorMessage(List unmatched) { public boolean isUnknownOption() { return isUnknownOption(unmatched, getCommandLine()); } /** Returns {@code true} and prints suggested solutions to the specified stream if such solutions exist, otherwise returns {@code false}. * @since 3.3.0 */ - public boolean printSuggestions(PrintStream out) { return printSuggestions(new PrintWriter(out, true)); } + public boolean printSuggestions(PrintStream out) { return printSuggestions(newPrintWriter(out, getStdoutEncoding())); } /** Returns {@code true} and prints suggested solutions to the specified stream if such solutions exist, otherwise returns {@code false}. * @since 4.0 */ public boolean printSuggestions(PrintWriter writer) { diff --git a/distributions/debian/pom.xml b/distributions/debian/pom.xml index 359aeec..16963e4 100644 --- a/distributions/debian/pom.xml +++ b/distributions/debian/pom.xml @@ -280,95 +280,96 @@ - - - debian-package-release - - - ci.release - - - - - - - - debian-package-snapshot - - - ci.snapshot - - - - - - org.codehaus.mojo - exec-maven-plugin - - - dev-snapshots - - - - - - - - debian-package-publish - - - ci.deploy - - - - - - org.codehaus.mojo - exec-maven-plugin - - - ${bintray.organization} - ${debian.package} - ${debian.package.version}${debian.package.version.suffix} - ${debian.architecture},i386,x86_64 - - - - - publish-debian-buster-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${debian.package}-debian-buster.deb - deb - buster,jessie,stretch - - - - - publish-ubuntu-focal-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${debian.package}-ubuntu-focal.deb - ubuntu - focal - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/distributions/docker-builders/Dockerfile-nativeimage b/distributions/docker-builders/Dockerfile-nativeimage index d06225e..d92e389 100644 --- a/distributions/docker-builders/Dockerfile-nativeimage +++ b/distributions/docker-builders/Dockerfile-nativeimage @@ -3,4 +3,17 @@ ARG FROM=ghcr.io/graalvm/graalvm-ce:java8-21.2.0 FROM ${FROM} # Install native-image tool -RUN gu install --no-progress native-image \ No newline at end of file +RUN gu install --no-progress native-image + +# Install musl +RUN curl -L -o x86_64-linux-musl-native.tgz http://musl.cc/x86_64-linux-musl-native.tgz \ + && tar -xvf x86_64-linux-musl-native.tgz -C /usr/local/bin \ + && export CC=/usr/local/bin/x86_64-linux-musl-native/bin/gcc \ + && curl -kL -o zlib-1.2.11.tar.gz https://zlib.net/zlib-1.2.11.tar.gz \ + && tar -xvf zlib-1.2.11.tar.gz \ + && cd zlib-1.2.11 \ + && ./configure --prefix=${WORKDIR}/x86_64-linux-musl-native --static \ + && make \ + && make install + +ENV PATH="${PATH}:${WORKDIR}/x86_64-linux-musl-native/bin" \ No newline at end of file diff --git a/distributions/docker/Dockerfile b/distributions/docker/Dockerfile index 5b3d26b..168bf95 100644 --- a/distributions/docker/Dockerfile +++ b/distributions/docker/Dockerfile @@ -1,23 +1,17 @@ -# glibc is required for proper DNS resolution within app -FROM busybox:1.35.0-glibc +FROM scratch -# Dynamically pass in name ARG NAME="sourcehawk" -# Setup user and group -ENV GROUP=${NAME} USER=${NAME} -RUN addgroup ${GROUP} && adduser -h /home/${USER} -G ${NAME} -D ${USER} +# Copy Group and User Files +COPY "/etc" "/etc" # Copy the native image executable into the image -ARG NATIVE_IMAGE_PATH -COPY --chown=${GROUP}:${USER} ${NATIVE_IMAGE_PATH} /usr/bin/sourcehawk - -# Give the native image executable permissions -RUN chmod +x /usr/bin/${NAME} +COPY --chown="${NAME}":"${NAME}" "target/native-image" "/entrypoint" # Set the user and working directory -USER ${USER} -WORKDIR /home/${USER} +USER "${NAME}" +WORKDIR "/work" # Set the native image as the entrypoint -ENTRYPOINT ["/usr/bin/sourcehawk"] \ No newline at end of file +CMD ["--help"] +ENTRYPOINT ["/entrypoint"] \ No newline at end of file diff --git a/distributions/docker/README.md b/distributions/docker/README.md index ec3aa6d..7528041 100644 --- a/distributions/docker/README.md +++ b/distributions/docker/README.md @@ -1,15 +1,15 @@ -# Sourcehawk Alpine Docker Image +# Sourcehawk Docker Image ### Scanning from local directory ```shell script -docker run -v "$(pwd):/home/sourcehawk" optumopensource/sourcehawk:1.0.0-alpine +docker run --rm -v $PWD:/work optumopensource/sourcehawk ``` Or with a custom working directory: ```shell script -docker run -v "$(pwd):/tmp" -w /tmp optumopensource/sourcehawk:1.0.0-alpine +docker run --rm -v $PWD:/tmp -w /tmp optumopensource/sourcehawk ``` The volume mounting is necessary in order to give the container access to the files to scan. \ No newline at end of file diff --git a/distributions/docker/etc/group b/distributions/docker/etc/group new file mode 100644 index 0000000..0d9e16b --- /dev/null +++ b/distributions/docker/etc/group @@ -0,0 +1 @@ +sourcehawk:x:1000:sourcehawk \ No newline at end of file diff --git a/distributions/docker/etc/passwd b/distributions/docker/etc/passwd new file mode 100644 index 0000000..f96df68 --- /dev/null +++ b/distributions/docker/etc/passwd @@ -0,0 +1 @@ +sourcehawk:x:1000:1000:Sourcehawk User,,,:/home/sourcehawk:/entrypoint \ No newline at end of file diff --git a/distributions/docker/pom.xml b/distributions/docker/pom.xml index bb59e6f..2b33223 100644 --- a/distributions/docker/pom.xml +++ b/distributions/docker/pom.xml @@ -61,6 +61,30 @@ + + + org.codehaus.mojo + exec-maven-plugin + + + + + update-native-image-permissions + prepare-package + + exec + + + chmod + + +x + target/native-image + + + + + + com.spotify @@ -76,10 +100,6 @@ ${docker.repository} ${docker.tag} - - ${global.project.name} - target/native-image - true @@ -103,6 +123,7 @@ docker run + --rm ${docker.repository}:${docker.tag} --version @@ -119,6 +140,7 @@ ${maven.multiModuleProjectDirectory} run + --rm -v ${maven.multiModuleProjectDirectory}:/home/sourcehawk ${docker.repository}:${docker.tag} diff --git a/distributions/linux/native-image-builder/Dockerfile b/distributions/linux/native-image-builder/Dockerfile index 2551249..59aefa9 100644 --- a/distributions/linux/native-image-builder/Dockerfile +++ b/distributions/linux/native-image-builder/Dockerfile @@ -1,4 +1,4 @@ -ARG GRAALVM_VERSION=21.2.0-java8 +ARG GRAALVM_VERSION=21.3.0-java11 FROM ghcr.io/optum/ci/nativeimage:graalvm-ce-${GRAALVM_VERSION} # Build Arguments @@ -17,4 +17,5 @@ RUN native-image -cp native-image.jar \ -H:+ReportExceptionStackTraces \ --report-unsupported-elements-at-runtime \ --no-fallback \ + --libc=musl \ --static \ No newline at end of file diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index 610b8ea..855bc85 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -193,12 +193,55 @@ + + + windows + + + windows + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + true + + + + + + + + + mac + + + mac + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + true + + + + + + linux-integration-tests unix + Linux @@ -229,17 +272,6 @@ - - - native-image-build-java8 - - 8 - - - 21.2.0-java8 - - - native-image-build-java11 @@ -255,7 +287,7 @@ native-image-build-java17 - 11 + 17 21.3.0-java17 diff --git a/distributions/rpm/pom.xml b/distributions/rpm/pom.xml index 10bac32..6dfbb23 100644 --- a/distributions/rpm/pom.xml +++ b/distributions/rpm/pom.xml @@ -193,23 +193,6 @@ - - build-and-tag-fedora-32 - prepare-package - - build - tag - - - ${project.artifactId}-builder-fedora-32 - ${project.version} - - fedora32 - /tmp/${rpm.package}-fedora-32.rpm - - true - - build-and-tag-fedora-33 prepare-package @@ -314,21 +297,6 @@ - - extract-fedora-32-package - package - - exec - - - ${project.parent.basedir}/scripts/extract-file-from-docker-container.sh - - ${project.artifactId}-builder-fedora-32:${project.version} - /tmp/${rpm.package}-fedora-32.rpm - ${project.build.directory} - - - extract-fedora-33-package package @@ -413,11 +381,6 @@ el8.${rpm.package.architecture} rpm - - ${project.build.directory}/${rpm.package}-fedora-32.rpm - fc32.${rpm.package.architecture} - rpm - ${project.build.directory}/${rpm.package}-fedora-33.rpm fc33.${rpm.package.architecture} @@ -442,156 +405,142 @@ - - - rpm-package-release - - - ci.release - - - - 1 - - - - rpm-package-snapshot - - - ci.snapshot - - - - - - org.codehaus.mojo - exec-maven-plugin - - - dev-snapshots - - - - - - - - rpm-package-publish - - - ci.deploy - - - - - - org.codehaus.mojo - exec-maven-plugin - - - ${bintray.organization} - ${rpm.package} - ${rpm.package.version} - ${rpm.package.release} - ${rpm.package.architecture} - - - - - publish-centos-7-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-centos-7.rpm - centos - el7 - - - - - publish-centos-8-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-centos-8.rpm - centos - el8 - - - - - publish-fedora-32-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-fedora-32.rpm - fedora - fc32 - - - - - publish-fedora-33-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-fedora-33.rpm - fedora - fc33 - - - - - publish-fedora-34-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-fedora-34.rpm - fedora - fc34 - - - - - publish-fedora-35-package - - exec - - deploy - - ${project.basedir}/scripts/publish-package-version-to-bintray.sh - - ${project.build.directory}/${rpm.package}-fedora-35.rpm - fedora - fc35 - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/distributions/scripts/build-and-push-docker-builders.sh b/distributions/scripts/build-and-push-docker-builders.sh index b0996ab..3bdf453 100755 --- a/distributions/scripts/build-and-push-docker-builders.sh +++ b/distributions/scripts/build-and-push-docker-builders.sh @@ -19,25 +19,21 @@ REGISTRY="ghcr.io" echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin $REGISTRY # Native Image -docker build -t $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.2.0-java8 -f "$DOCKER_BUILDERS_DIR/Dockerfile-nativeimage" --build-arg FROM=ghcr.io/graalvm/graalvm-ce:java8-21.2.0 . docker build -t $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java11 -f "$DOCKER_BUILDERS_DIR/Dockerfile-nativeimage" --build-arg FROM=ghcr.io/graalvm/graalvm-ce:java11-21.3.0 . docker build -t $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java17 -f "$DOCKER_BUILDERS_DIR/Dockerfile-nativeimage" --build-arg FROM=ghcr.io/graalvm/graalvm-ce:java17-21.3.0 . # RPM Build docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:centos7 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=centos:7 . docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:centos8 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=centos:8 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora32 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:32 . docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora33 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:33 . docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora34 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:34 . docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora35 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:35 . # Push All Builders to Remote Registry -docker push $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.2.0-java8 docker push $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java11 docker push $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java17 docker push $REGISTRY/$DOCKER_ORG/rpmbuild:centos7 docker push $REGISTRY/$DOCKER_ORG/rpmbuild:centos8 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora32 docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora33 docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora34 docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora35 From 0c75dfe4a7e5c19392878b759095f800de7eb1bb Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 16:24:52 -0500 Subject: [PATCH 14/35] remote-repository: Update github workflows --- .github/workflows/maven-ci.yml | 12 ++++-------- .github/workflows/release.yml | 8 ++++---- distributions/docker-builders/Dockerfile-nativeimage | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index 5ff5f60..d96bce6 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -130,24 +130,20 @@ jobs: path: build - uses: actions/download-artifact@v2 with: - name: bash-completion-script-8 + name: bash-completion-script-11 path: build - uses: actions/download-artifact@v2 with: - name: manpages-8 + name: manpages-11 path: build - name: Rename Native Image JAR working-directory: build run: mv *.jar native-image.jar - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: 8 - name: Setup GraalVM uses: graalvm/setup-graalvm@v1 with: - version: '21.2.0' - java-version: '8' + version: '21.3.0' + java-version: '11' components: 'native-image' github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Mac Native Image diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfa76b4..8069a29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -247,7 +247,7 @@ jobs: steps: - uses: actions/download-artifact@v2 with: - name: native-image + name: native-image-java11 path: build - uses: actions/download-artifact@v2 with: @@ -263,12 +263,12 @@ jobs: - name: Setup Java uses: actions/setup-java@v1 with: - java-version: 8 + java-version: 11 - name: Setup GraalVM uses: graalvm/setup-graalvm@v1 with: - version: '21.2.0' - java-version: '8' + version: '21.3.0' + java-version: '11' components: 'native-image' github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Mac Native Image diff --git a/distributions/docker-builders/Dockerfile-nativeimage b/distributions/docker-builders/Dockerfile-nativeimage index d92e389..1bc2782 100644 --- a/distributions/docker-builders/Dockerfile-nativeimage +++ b/distributions/docker-builders/Dockerfile-nativeimage @@ -1,5 +1,5 @@ # Oracle GraalVM Java 8 Base Container -ARG FROM=ghcr.io/graalvm/graalvm-ce:java8-21.2.0 +ARG FROM=ghcr.io/graalvm/graalvm-ce:java11-21.3.0 FROM ${FROM} # Install native-image tool From 80531bc331b562c05f9b02fffc986936b5dcbdc7 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 16:39:52 -0500 Subject: [PATCH 15/35] remote-repository: Fix native image builder and push to ghcr --- .../docker-builders/Dockerfile-nativeimage | 6 +-- distributions/docker/pom.xml | 47 +++++++++++++++++-- ...-to-docker-hub.sh => push-docker-image.sh} | 0 3 files changed, 44 insertions(+), 9 deletions(-) rename distributions/docker/scripts/{push-docker-image-to-docker-hub.sh => push-docker-image.sh} (100%) diff --git a/distributions/docker-builders/Dockerfile-nativeimage b/distributions/docker-builders/Dockerfile-nativeimage index 1bc2782..2056017 100644 --- a/distributions/docker-builders/Dockerfile-nativeimage +++ b/distributions/docker-builders/Dockerfile-nativeimage @@ -5,15 +5,13 @@ FROM ${FROM} # Install native-image tool RUN gu install --no-progress native-image -# Install musl +# Install musl libc RUN curl -L -o x86_64-linux-musl-native.tgz http://musl.cc/x86_64-linux-musl-native.tgz \ && tar -xvf x86_64-linux-musl-native.tgz -C /usr/local/bin \ && export CC=/usr/local/bin/x86_64-linux-musl-native/bin/gcc \ && curl -kL -o zlib-1.2.11.tar.gz https://zlib.net/zlib-1.2.11.tar.gz \ && tar -xvf zlib-1.2.11.tar.gz \ && cd zlib-1.2.11 \ - && ./configure --prefix=${WORKDIR}/x86_64-linux-musl-native --static \ + && ./configure --prefix=/usr/local/bin/x86_64-linux-musl-native --static \ && make \ && make install - -ENV PATH="${PATH}:${WORKDIR}/x86_64-linux-musl-native/bin" \ No newline at end of file diff --git a/distributions/docker/pom.xml b/distributions/docker/pom.xml index 2b33223..1a11871 100644 --- a/distributions/docker/pom.xml +++ b/distributions/docker/pom.xml @@ -19,6 +19,7 @@ ${project.version} + ghcr.io/optum/sourcehawk @@ -91,10 +92,21 @@ dockerfile-maven-plugin - build-and-tag + build prepare-package build + + + ${docker.repository} + ${docker.tag} + true + + + + tag-docker-hub + prepare-package + tag @@ -103,6 +115,18 @@ true + + tag-github-container-registry + prepare-package + + tag + + + ${ghcr.repository} + ${docker.tag} + true + + @@ -142,7 +166,7 @@ run --rm -v - ${maven.multiModuleProjectDirectory}:/home/sourcehawk + ${maven.multiModuleProjectDirectory}:/work ${docker.repository}:${docker.tag} scan -f @@ -159,7 +183,7 @@ - docker-hub-push + docker-push ci.release @@ -172,18 +196,31 @@ exec-maven-plugin - deploy-docker-image + push-docker-image-to-docker-hub deploy exec - ${project.basedir}/scripts/push-docker-image-to-docker-hub.sh + ${project.basedir}/scripts/push-docker-image.sh ${docker.repository}:${docker.tag} + + push-docker-image-to-github-container-registry + deploy + + exec + + + ${project.basedir}/scripts/push-docker-image.sh + + ${ghcr.repository}:${docker.tag} + + + diff --git a/distributions/docker/scripts/push-docker-image-to-docker-hub.sh b/distributions/docker/scripts/push-docker-image.sh similarity index 100% rename from distributions/docker/scripts/push-docker-image-to-docker-hub.sh rename to distributions/docker/scripts/push-docker-image.sh From 1ddfcf724431adec9491b9c447e4a138497039ad Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 17:40:15 -0500 Subject: [PATCH 16/35] remote-repository: Fix docker image builder for native image musl, and use github actions for docker layer caching --- .../build-and-push-docker-builders.yml | 103 +++++++++++++++++- .github/workflows/maven-ci.yml | 6 - .github/workflows/release.yml | 6 - .../docker-builders/Dockerfile-nativeimage | 19 ++-- .../scripts/build-and-push-docker-builders.sh | 42 ------- 5 files changed, 108 insertions(+), 68 deletions(-) delete mode 100755 distributions/scripts/build-and-push-docker-builders.sh diff --git a/.github/workflows/build-and-push-docker-builders.yml b/.github/workflows/build-and-push-docker-builders.yml index 2793019..ca8510d 100644 --- a/.github/workflows/build-and-push-docker-builders.yml +++ b/.github/workflows/build-and-push-docker-builders.yml @@ -13,9 +13,100 @@ jobs: - uses: actions/checkout@v2 with: ref: ${{ github.event.inputs.ref }} - - name: Build and Push Docker Images to Github Container Registry - shell: bash - run: ./distributions/scripts/build-and-push-docker-builders.sh - env: - DOCKER_USERNAME: ${{ secrets.GIT_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.GIT_PACKAGE_TOKEN }} + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker Layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Login to Github Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GIT_USERNAME }} + password: ${{ secrets.GIT_PACKAGE_TOKEN }} + - name: Build and Push graalvm-ce-21.3.0-java11 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-nativeimage + build-args: FROM_VERSION=java11-21.3.0 + push: false + tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java11 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + - name: Build and Push graalvm-ce-21.3.0-java17 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-nativeimage + build-args: FROM_VERSION=java17-21.3.0 + push: false + tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java17 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + - name: Build and Push rpmbuild centos7 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=centos:7 + push: false + tags: ghcr.io/optum/ci/rpmbuild:centos7 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + - name: Build and Push rpmbuild centos8 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=centos:8 + push: false + tags: ghcr.io/optum/ci/rpmbuild:centos8 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + - name: Build and Push rpmbuild fedora33 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=fedora:33 + push: false + tags: ghcr.io/optum/ci/rpmbuild:fedora33 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + - name: Build and Push rpmbuild fedora34 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=fedora:34 + push: false + tags: ghcr.io/optum/ci/rpmbuild:fedora34 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + - name: Build and Push rpmbuild fedora35 + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: distributions/docker-builders/Dockerfile-rpmbuild + build-args: FROM=fedora:35 + push: false + tags: ghcr.io/optum/ci/rpmbuild:fedora35 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + - name: Relocate Docker Layer Cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index d96bce6..1e92448 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -61,12 +61,6 @@ jobs: server-password: SONATYPE_PASSWORD gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} gpg-passphrase: SONATYPE_GPG_PASSPHRASE - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GIT_USERNAME }} - password: ${{ secrets.GIT_PACKAGE_TOKEN }} - name: Set Maven Project Version shell: bash run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8069a29..f380e2c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,12 +37,6 @@ jobs: server-password: SONATYPE_PASSWORD gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} gpg-passphrase: SONATYPE_GPG_PASSPHRASE - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GIT_USERNAME }} - password: ${{ secrets.GIT_PACKAGE_TOKEN }} - name: Set Maven Project Version id: set_maven_project_version shell: bash diff --git a/distributions/docker-builders/Dockerfile-nativeimage b/distributions/docker-builders/Dockerfile-nativeimage index 2056017..061ea3c 100644 --- a/distributions/docker-builders/Dockerfile-nativeimage +++ b/distributions/docker-builders/Dockerfile-nativeimage @@ -1,17 +1,20 @@ -# Oracle GraalVM Java 8 Base Container -ARG FROM=ghcr.io/graalvm/graalvm-ce:java11-21.3.0 -FROM ${FROM} +# GraalVM Community Edition Base Container +ARG FROM_VERSION=java11-21.3.0 +FROM ghcr.io/graalvm/graalvm-ce:${FROM_VERSION} # Install native-image tool RUN gu install --no-progress native-image # Install musl libc RUN curl -L -o x86_64-linux-musl-native.tgz http://musl.cc/x86_64-linux-musl-native.tgz \ - && tar -xvf x86_64-linux-musl-native.tgz -C /usr/local/bin \ - && export CC=/usr/local/bin/x86_64-linux-musl-native/bin/gcc \ - && curl -kL -o zlib-1.2.11.tar.gz https://zlib.net/zlib-1.2.11.tar.gz \ - && tar -xvf zlib-1.2.11.tar.gz \ + && tar -xvf x86_64-linux-musl-native.tgz -C /opt \ + && export CC=/opt/x86_64-linux-musl-native/bin/gcc \ + && curl -kL -o zlib.tar.gz https://zlib.net/zlib-1.2.11.tar.gz \ + && tar -xvf zlib.tar.gz \ && cd zlib-1.2.11 \ - && ./configure --prefix=/usr/local/bin/x86_64-linux-musl-native --static \ + && ./configure --prefix=/opt/x86_64-linux-musl-native --static \ && make \ && make install + +# Update PATH Environment Variable +ENV PATH="$PATH:/opt/x86_64-linux-musl-native/bin" diff --git a/distributions/scripts/build-and-push-docker-builders.sh b/distributions/scripts/build-and-push-docker-builders.sh deleted file mode 100755 index 3bdf453..0000000 --- a/distributions/scripts/build-and-push-docker-builders.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -############################################################################################################## -# -# Push Docker Builders to Remote Registry -# -############################################################################################################## - -set -e - -ROOT_DIR="$( cd "$( dirname "$( dirname "$( dirname "${BASH_SOURCE[0]}" )")")" && pwd )" -DOCKER_BUILDERS_DIR="$ROOT_DIR/distributions/docker-builders" - -# Variables -DOCKER_ORG="optum/ci" -REGISTRY="ghcr.io" - -# Login to Registry -echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin $REGISTRY - -# Native Image -docker build -t $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java11 -f "$DOCKER_BUILDERS_DIR/Dockerfile-nativeimage" --build-arg FROM=ghcr.io/graalvm/graalvm-ce:java11-21.3.0 . -docker build -t $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java17 -f "$DOCKER_BUILDERS_DIR/Dockerfile-nativeimage" --build-arg FROM=ghcr.io/graalvm/graalvm-ce:java17-21.3.0 . - -# RPM Build -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:centos7 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=centos:7 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:centos8 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=centos:8 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora33 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:33 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora34 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:34 . -docker build -t $REGISTRY/$DOCKER_ORG/rpmbuild:fedora35 -f "$DOCKER_BUILDERS_DIR/Dockerfile-rpmbuild" --build-arg FROM=fedora:35 . - -# Push All Builders to Remote Registry -docker push $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java11 -docker push $REGISTRY/$DOCKER_ORG/nativeimage:graalvm-ce-21.3.0-java17 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:centos7 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:centos8 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora33 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora34 -docker push $REGISTRY/$DOCKER_ORG/rpmbuild:fedora35 - -# Log out of registry -docker logout $REGISTRY \ No newline at end of file From 35bfe1b299fe6971b63eb697a6a8d18149e43567 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 17:51:44 -0500 Subject: [PATCH 17/35] remote-repository: Push docker builders --- .github/workflows/build-and-push-docker-builders.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build-and-push-docker-builders.yml b/.github/workflows/build-and-push-docker-builders.yml index ca8510d..4c231b6 100644 --- a/.github/workflows/build-and-push-docker-builders.yml +++ b/.github/workflows/build-and-push-docker-builders.yml @@ -36,7 +36,6 @@ jobs: context: . file: distributions/docker-builders/Dockerfile-nativeimage build-args: FROM_VERSION=java11-21.3.0 - push: false tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java11 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new @@ -47,7 +46,6 @@ jobs: context: . file: distributions/docker-builders/Dockerfile-nativeimage build-args: FROM_VERSION=java17-21.3.0 - push: false tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java17 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new @@ -58,7 +56,6 @@ jobs: context: . file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=centos:7 - push: false tags: ghcr.io/optum/ci/rpmbuild:centos7 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new @@ -69,7 +66,6 @@ jobs: context: . file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=centos:8 - push: false tags: ghcr.io/optum/ci/rpmbuild:centos8 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new @@ -80,7 +76,6 @@ jobs: context: . file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=fedora:33 - push: false tags: ghcr.io/optum/ci/rpmbuild:fedora33 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new @@ -91,7 +86,6 @@ jobs: context: . file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=fedora:34 - push: false tags: ghcr.io/optum/ci/rpmbuild:fedora34 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new @@ -102,7 +96,6 @@ jobs: context: . file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=fedora:35 - push: false tags: ghcr.io/optum/ci/rpmbuild:fedora35 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new From 495043342b01189f1bc0499b5738c135b449fc75 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 17:58:25 -0500 Subject: [PATCH 18/35] remote-repository: Push docker builders --- .github/workflows/build-and-push-docker-builders.yml | 10 +++++++++- distributions/docker-builders/Dockerfile-nativeimage | 8 ++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-push-docker-builders.yml b/.github/workflows/build-and-push-docker-builders.yml index 4c231b6..8a62a1a 100644 --- a/.github/workflows/build-and-push-docker-builders.yml +++ b/.github/workflows/build-and-push-docker-builders.yml @@ -10,7 +10,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Git Checkout + uses: actions/checkout@v2 with: ref: ${{ github.event.inputs.ref }} - name: Set up Docker Buildx @@ -37,6 +38,7 @@ jobs: file: distributions/docker-builders/Dockerfile-nativeimage build-args: FROM_VERSION=java11-21.3.0 tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java11 + push: true cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push graalvm-ce-21.3.0-java17 @@ -47,6 +49,7 @@ jobs: file: distributions/docker-builders/Dockerfile-nativeimage build-args: FROM_VERSION=java17-21.3.0 tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java17 + push: true cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild centos7 @@ -57,6 +60,7 @@ jobs: file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=centos:7 tags: ghcr.io/optum/ci/rpmbuild:centos7 + push: true cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild centos8 @@ -67,6 +71,7 @@ jobs: file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=centos:8 tags: ghcr.io/optum/ci/rpmbuild:centos8 + push: true cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild fedora33 @@ -77,6 +82,7 @@ jobs: file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=fedora:33 tags: ghcr.io/optum/ci/rpmbuild:fedora33 + push: true cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild fedora34 @@ -87,6 +93,7 @@ jobs: file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=fedora:34 tags: ghcr.io/optum/ci/rpmbuild:fedora34 + push: true cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild fedora35 @@ -97,6 +104,7 @@ jobs: file: distributions/docker-builders/Dockerfile-rpmbuild build-args: FROM=fedora:35 tags: ghcr.io/optum/ci/rpmbuild:fedora35 + push: true cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Relocate Docker Layer Cache diff --git a/distributions/docker-builders/Dockerfile-nativeimage b/distributions/docker-builders/Dockerfile-nativeimage index 061ea3c..3b2105e 100644 --- a/distributions/docker-builders/Dockerfile-nativeimage +++ b/distributions/docker-builders/Dockerfile-nativeimage @@ -6,11 +6,11 @@ FROM ghcr.io/graalvm/graalvm-ce:${FROM_VERSION} RUN gu install --no-progress native-image # Install musl libc -RUN curl -L -o x86_64-linux-musl-native.tgz http://musl.cc/x86_64-linux-musl-native.tgz \ - && tar -xvf x86_64-linux-musl-native.tgz -C /opt \ +RUN curl -ksL -o x86_64-linux-musl-native.tgz http://musl.cc/x86_64-linux-musl-native.tgz \ + && tar -xf x86_64-linux-musl-native.tgz -C /opt \ && export CC=/opt/x86_64-linux-musl-native/bin/gcc \ - && curl -kL -o zlib.tar.gz https://zlib.net/zlib-1.2.11.tar.gz \ - && tar -xvf zlib.tar.gz \ + && curl -ksL -o zlib.tar.gz https://zlib.net/zlib-1.2.11.tar.gz \ + && tar -xf zlib.tar.gz \ && cd zlib-1.2.11 \ && ./configure --prefix=/opt/x86_64-linux-musl-native --static \ && make \ From a0a57ac86b3f5389ff0d7f3a2a00eabf69033fe1 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 18:21:42 -0500 Subject: [PATCH 19/35] remote-repository: 0.7.0 --- bom/pom.xml | 2 +- cli/pom.xml | 2 +- core/pom.xml | 2 +- distributions/debian/pom.xml | 2 +- distributions/docker/pom.xml | 2 +- distributions/linux/pom.xml | 2 +- distributions/pom.xml | 2 +- distributions/rpm/pom.xml | 2 +- enforcer/core/pom.xml | 2 +- enforcer/file/aot/pom.xml | 2 +- enforcer/file/common/pom.xml | 2 +- enforcer/file/core/pom.xml | 2 +- enforcer/file/docker/pom.xml | 2 +- enforcer/file/maven/pom.xml | 2 +- enforcer/file/pom.xml | 2 +- enforcer/file/registry/pom.xml | 2 +- enforcer/pom.xml | 2 +- exec/pom.xml | 2 +- pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index c18379b..febfdc6 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT sourcehawk-bom diff --git a/cli/pom.xml b/cli/pom.xml index 624debf..7ac24e7 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -8,7 +8,7 @@ sourcehawk com.optum.sourcehawk - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT sourcehawk-cli diff --git a/core/pom.xml b/core/pom.xml index a840223..0d73a61 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/distributions/debian/pom.xml b/distributions/debian/pom.xml index 16963e4..158df2d 100644 --- a/distributions/debian/pom.xml +++ b/distributions/debian/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-dist - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/distributions/docker/pom.xml b/distributions/docker/pom.xml index 1a11871..2485517 100644 --- a/distributions/docker/pom.xml +++ b/distributions/docker/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-dist - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index 855bc85..423b4a1 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-dist - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/distributions/pom.xml b/distributions/pom.xml index e1a410e..ebae0b5 100644 --- a/distributions/pom.xml +++ b/distributions/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/distributions/rpm/pom.xml b/distributions/rpm/pom.xml index 6dfbb23..19e90da 100644 --- a/distributions/rpm/pom.xml +++ b/distributions/rpm/pom.xml @@ -8,7 +8,7 @@ sourcehawk-dist com.optum.sourcehawk - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT sourcehawk-dist-rpm diff --git a/enforcer/core/pom.xml b/enforcer/core/pom.xml index 93e09a2..fb9c5d6 100644 --- a/enforcer/core/pom.xml +++ b/enforcer/core/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/aot/pom.xml b/enforcer/file/aot/pom.xml index 7c233bf..43086e6 100644 --- a/enforcer/file/aot/pom.xml +++ b/enforcer/file/aot/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/common/pom.xml b/enforcer/file/common/pom.xml index 8d0ef8a..8604ff5 100644 --- a/enforcer/file/common/pom.xml +++ b/enforcer/file/common/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/core/pom.xml b/enforcer/file/core/pom.xml index cd74cb7..96c8013 100644 --- a/enforcer/file/core/pom.xml +++ b/enforcer/file/core/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/docker/pom.xml b/enforcer/file/docker/pom.xml index 988a7bc..6ec6ae9 100644 --- a/enforcer/file/docker/pom.xml +++ b/enforcer/file/docker/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/maven/pom.xml b/enforcer/file/maven/pom.xml index 699b4cf..76dd2ef 100644 --- a/enforcer/file/maven/pom.xml +++ b/enforcer/file/maven/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/pom.xml b/enforcer/file/pom.xml index 1852619..bbb6251 100644 --- a/enforcer/file/pom.xml +++ b/enforcer/file/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/file/registry/pom.xml b/enforcer/file/registry/pom.xml index bf8c359..663031c 100644 --- a/enforcer/file/registry/pom.xml +++ b/enforcer/file/registry/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk-enforcer-file - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/enforcer/pom.xml b/enforcer/pom.xml index 6254ec3..ee974cc 100644 --- a/enforcer/pom.xml +++ b/enforcer/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/exec/pom.xml b/exec/pom.xml index 1033150..a7bc559 100644 --- a/exec/pom.xml +++ b/exec/pom.xml @@ -8,7 +8,7 @@ com.optum.sourcehawk sourcehawk - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index ab8b605..0aae6ea 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ sourcehawk - 0.6.1-SNAPSHOT + 0.7.0-SNAPSHOT pom Sourcehawk From c61d4318872abf54f0e5ac649ab7142a16bac82a Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 18:33:27 -0500 Subject: [PATCH 20/35] remote-repository: Skip native image build for java 8 --- distributions/linux/pom.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index 423b4a1..9adab8d 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -23,6 +23,7 @@ /home/${global.project.name} ${docker.workdir}/${global.project.name} ${project.build.directory}/${global.project.name} + false true @@ -57,6 +58,7 @@ copy-resources + ${native.image.build.skip} ${project.parent.parent.basedir}/gh-pages/manpages @@ -82,6 +84,7 @@ copy-dependencies + ${native.image.build.skip} com.optum.sourcehawk sourcehawk-cli native-image @@ -95,6 +98,7 @@ copy + ${native.image.build.skip} com.optum.sourcehawk @@ -123,6 +127,7 @@ tag + ${native.image.build.skip} ${basedir}/native-image-builder/Dockerfile ${docker.image} ${project.version} @@ -152,6 +157,7 @@ exec + ${native.image.build.skip} ../scripts/extract-file-from-docker-container.sh ${docker.image}:${project.version} @@ -176,6 +182,7 @@ post-integration-test + ${native.image.build.skip} ${native.image} @@ -259,6 +266,7 @@ exec + ${native.image.build.skip} ${native.image} --version @@ -272,6 +280,17 @@ + + + native-image-build-java11 + + 8 + + + true + + + native-image-build-java11 From e23b5f1167d1045379ec7c07f9568ca1084999f7 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 18:35:12 -0500 Subject: [PATCH 21/35] remote-repository: Skip native image build for java 8 --- distributions/linux/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index 9adab8d..d6fb4ee 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -282,7 +282,7 @@ - native-image-build-java11 + java8 8 @@ -293,7 +293,7 @@ - native-image-build-java11 + java11 11 @@ -304,7 +304,7 @@ - native-image-build-java17 + java17 17 From e611c42be46d53d76e9ee863f45259aa11b85eaf Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 18:51:41 -0500 Subject: [PATCH 22/35] remote-repository: Fix java8 build and workflow dependencies --- .github/workflows/maven-ci.yml | 4 ++-- .github/workflows/release.yml | 20 +++++++++---------- cli/scripts/update-picocli.sh | 2 +- cli/src/main/java/picocli/CommandLine.java | 1 + .../docker-builders/Dockerfile-nativeimage | 2 +- distributions/linux/pom.xml | 4 +++- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index 1e92448..da5ac95 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -116,7 +116,7 @@ jobs: report_paths: '**/target/surefire-reports/TEST-*.xml,**/target/failsafe-reports/TEST-*.xml' build-mac-native-image: runs-on: macos-latest - needs: build + needs: build-11 steps: - uses: actions/download-artifact@v2 with: @@ -168,7 +168,7 @@ jobs: run: ./sourcehawk help build-windows-native-image: runs-on: windows-latest - needs: build + needs: build-11 steps: - uses: actions/download-artifact@v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f380e2c..b94e170 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -237,7 +237,7 @@ jobs: path: cli/target/*-native-image.jar build-mac-native-image: runs-on: macos-latest - needs: build + needs: build-11 steps: - uses: actions/download-artifact@v2 with: @@ -284,9 +284,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build.outputs.RELEASE_ASSET_UPLOAD_URL }} + upload_url: ${{ needs.build-11.outputs.RELEASE_ASSET_UPLOAD_URL }} asset_path: build/sourcehawk - asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-darwin-x86_64 + asset_name: sourcehawk-${{ needs.build-11.outputs.RELEASE_VERSION }}-darwin-x86_64 asset_content_type: application/octet-stream - name: Upload Sourcehawk Mac Homebrew Tap Formula Archive id: upload_homebrew_tap_formula_archive @@ -296,25 +296,25 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build.outputs.RELEASE_ASSET_UPLOAD_URL }} + upload_url: ${{ needs.build-11.outputs.RELEASE_ASSET_UPLOAD_URL }} asset_path: build/sourcehawk-homebrew-tap-formula.tar.gz - asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-darwin-x86_64.tar.gz + asset_name: sourcehawk-${{ needs.build-11.outputs.RELEASE_VERSION }}-darwin-x86_64.tar.gz asset_content_type: application/octet-stream - name: Update Optum Homebrew Tap Formula uses: mislav/bump-homebrew-formula-action@v1.10 continue-on-error: true with: formula-name: sourcehawk - tag-name: ${{ needs.build.outputs.RELEASE_VERSION }} + tag-name: ${{ needs.build-11.outputs.RELEASE_VERSION }} homebrew-tap: optum/homebrew-tap base-branch: main download-url: ${{ steps.upload_homebrew_tap_formula_archive.outputs.browser_download_url }} - commit-message: Updating sourcehawk formula to latest release version ${{ needs.build.outputs.RELEASE_VERSION }} + commit-message: Updating sourcehawk formula to latest release version ${{ needs.build-11.outputs.RELEASE_VERSION }} env: COMMITTER_TOKEN: ${{ secrets.GIT_COMMITTER_TOKEN }} build-windows-native-image: runs-on: windows-latest - needs: [build, build-java11] + needs: build-java11 steps: - uses: actions/download-artifact@v2 with: @@ -344,7 +344,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build.outputs.RELEASE_ASSET_UPLOAD_URL }} + upload_url: ${{ needs.build-11.outputs.RELEASE_ASSET_UPLOAD_URL }} asset_path: sourcehawk.exe - asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-windows-x86_64.exe + asset_name: sourcehawk-${{ needs.build-java11.outputs.RELEASE_VERSION }}-windows-x86_64.exe asset_content_type: application/octet-stream diff --git a/cli/scripts/update-picocli.sh b/cli/scripts/update-picocli.sh index 3d35f7d..84011f5 100755 --- a/cli/scripts/update-picocli.sh +++ b/cli/scripts/update-picocli.sh @@ -28,7 +28,7 @@ curl -ksf "$LICENSE_URL" > "$LICENSE_FILE_PATH" curl -ksf "$SOURCE_URL" > "$SOURCE_FILE_PATH" # Add some warning suppression to the java source file -sed -i.bak -e 's/public\sclass\sCommandLine/@SuppressWarnings({"rawtypes", "deprecation" })\npublic class CommandLine/g' \ +sed -i.bak -e 's/public class CommandLine/@SuppressWarnings({"rawtypes", "deprecation" })\npublic class CommandLine/g' \ -e 's/TODO/TIDO/g' "$SOURCE_FILE_PATH" \ && rm -rf "$SOURCE_FILE_PATH.bak" diff --git a/cli/src/main/java/picocli/CommandLine.java b/cli/src/main/java/picocli/CommandLine.java index a0f8454..07a29c9 100644 --- a/cli/src/main/java/picocli/CommandLine.java +++ b/cli/src/main/java/picocli/CommandLine.java @@ -142,6 +142,7 @@ * Classes Related to Parsing Command Line Arguments *

*/ +@SuppressWarnings({"rawtypes", "deprecation" }) public class CommandLine { /** This is picocli version {@value}. */ diff --git a/distributions/docker-builders/Dockerfile-nativeimage b/distributions/docker-builders/Dockerfile-nativeimage index 3b2105e..b0c52ca 100644 --- a/distributions/docker-builders/Dockerfile-nativeimage +++ b/distributions/docker-builders/Dockerfile-nativeimage @@ -6,7 +6,7 @@ FROM ghcr.io/graalvm/graalvm-ce:${FROM_VERSION} RUN gu install --no-progress native-image # Install musl libc -RUN curl -ksL -o x86_64-linux-musl-native.tgz http://musl.cc/x86_64-linux-musl-native.tgz \ +RUN curl -ksL -o x86_64-linux-musl-native.tgz https://more.musl.cc/10.2.1/x86_64-linux-musl/x86_64-linux-musl-native.tgz \ && tar -xf x86_64-linux-musl-native.tgz -C /opt \ && export CC=/opt/x86_64-linux-musl-native/bin/gcc \ && curl -ksL -o zlib.tar.gz https://zlib.net/zlib-1.2.11.tar.gz \ diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index d6fb4ee..effa34d 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -23,7 +23,7 @@ /home/${global.project.name} ${docker.workdir}/${global.project.name} ${project.build.directory}/${global.project.name} - false + true @@ -299,6 +299,7 @@ 21.3.0-java11 + false @@ -310,6 +311,7 @@ 21.3.0-java17 + false From 27f1624e5cb5fa0cfb2667c762eb600a547d6568 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 19:11:07 -0500 Subject: [PATCH 23/35] remote-repository: Fix needs syntax --- .github/workflows/maven-ci.yml | 5 +++-- .github/workflows/release.yml | 16 ++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index da5ac95..aa5dbaa 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -114,9 +114,10 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} report_paths: '**/target/surefire-reports/TEST-*.xml,**/target/failsafe-reports/TEST-*.xml' + check_name: Test Report - Java ${{ matrix.java }} build-mac-native-image: runs-on: macos-latest - needs: build-11 + needs: build steps: - uses: actions/download-artifact@v2 with: @@ -168,7 +169,7 @@ jobs: run: ./sourcehawk help build-windows-native-image: runs-on: windows-latest - needs: build-11 + needs: build steps: - uses: actions/download-artifact@v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b94e170..1af3a39 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -237,7 +237,7 @@ jobs: path: cli/target/*-native-image.jar build-mac-native-image: runs-on: macos-latest - needs: build-11 + needs: build-java11 steps: - uses: actions/download-artifact@v2 with: @@ -284,9 +284,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build-11.outputs.RELEASE_ASSET_UPLOAD_URL }} + upload_url: ${{ needs.build-java11.outputs.RELEASE_ASSET_UPLOAD_URL }} asset_path: build/sourcehawk - asset_name: sourcehawk-${{ needs.build-11.outputs.RELEASE_VERSION }}-darwin-x86_64 + asset_name: sourcehawk-${{ needs.build-java11.outputs.RELEASE_VERSION }}-darwin-x86_64 asset_content_type: application/octet-stream - name: Upload Sourcehawk Mac Homebrew Tap Formula Archive id: upload_homebrew_tap_formula_archive @@ -296,20 +296,20 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build-11.outputs.RELEASE_ASSET_UPLOAD_URL }} + upload_url: ${{ needs.build-java11.outputs.RELEASE_ASSET_UPLOAD_URL }} asset_path: build/sourcehawk-homebrew-tap-formula.tar.gz - asset_name: sourcehawk-${{ needs.build-11.outputs.RELEASE_VERSION }}-darwin-x86_64.tar.gz + asset_name: sourcehawk-${{ needs.build-java11.outputs.RELEASE_VERSION }}-darwin-x86_64.tar.gz asset_content_type: application/octet-stream - name: Update Optum Homebrew Tap Formula uses: mislav/bump-homebrew-formula-action@v1.10 continue-on-error: true with: formula-name: sourcehawk - tag-name: ${{ needs.build-11.outputs.RELEASE_VERSION }} + tag-name: ${{ needs.build-java11.outputs.RELEASE_VERSION }} homebrew-tap: optum/homebrew-tap base-branch: main download-url: ${{ steps.upload_homebrew_tap_formula_archive.outputs.browser_download_url }} - commit-message: Updating sourcehawk formula to latest release version ${{ needs.build-11.outputs.RELEASE_VERSION }} + commit-message: Updating sourcehawk formula to latest release version ${{ needs.build-java11.outputs.RELEASE_VERSION }} env: COMMITTER_TOKEN: ${{ secrets.GIT_COMMITTER_TOKEN }} build-windows-native-image: @@ -344,7 +344,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build-11.outputs.RELEASE_ASSET_UPLOAD_URL }} + upload_url: ${{ needs.build-java11.outputs.RELEASE_ASSET_UPLOAD_URL }} asset_path: sourcehawk.exe asset_name: sourcehawk-${{ needs.build-java11.outputs.RELEASE_VERSION }}-windows-x86_64.exe asset_content_type: application/octet-stream From 1758b928ae03a943f7eaf94f182f3b2ea5eae13a Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 19:21:58 -0500 Subject: [PATCH 24/35] remote-repository: 1.8 reference to identigy java 8 in mvn profile --- distributions/linux/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index effa34d..3479727 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -284,7 +284,7 @@ java8 - 8 + 1.8 true From 05be46b7ac134bcad13ea3dc67288d181b395e33 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 19:26:43 -0500 Subject: [PATCH 25/35] remote-repository: Skip failsafe integration tests for java8 --- .github/workflows/maven-ci.yml | 2 +- distributions/linux/pom.xml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index aa5dbaa..e061736 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -114,7 +114,7 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} report_paths: '**/target/surefire-reports/TEST-*.xml,**/target/failsafe-reports/TEST-*.xml' - check_name: Test Report - Java ${{ matrix.java }} + check_name: Test Report (${{ matrix.java }}) build-mac-native-image: runs-on: macos-latest needs: build diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index 3479727..63fff97 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -289,6 +289,17 @@ true + + + + org.apache.maven.plugins + maven-failsafe-plugin + + true + + + + From 8b6d5c694fe44724ebefadadd6d8661d779e709b Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 19:55:23 -0500 Subject: [PATCH 26/35] remote-repository: Only build dist when not java 8 --- .github/workflows/release.yml | 108 ++++++++++++++++------------------ distributions/linux/pom.xml | 32 ---------- distributions/pom.xml | 7 --- pom.xml | 1 + 4 files changed, 52 insertions(+), 96 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1af3a39..a5ea1fd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,23 +56,45 @@ jobs: SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} SONATYPE_GPG_PASSPHRASE: ${{ secrets.SONATYPE_GPG_PASSPHRASE }} + build-java11: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.ref }} + - name: Setup Java and Maven + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Set Maven Project Version + shell: bash + run: | + RELEASE_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | tail -1 | tr -d '\r\n') + [[ "$RELEASE_VERSION" == *"-SNAPSHOT" ]] && RELEASE_VERSION="${RELEASE_VERSION%"-SNAPSHOT"}" && ./mvnw --batch-mode versions:set -D removeSnapshot || true + [[ "$(git tag -l $RELEASE_VERSION)" == "$RELEASE_VERSION" ]] && echo "Tag $RELEASE_VERSION already exists" && exit 1 + echo ::set-output name=RELEASE_VERSION::$RELEASE_VERSION + - name: Build Maven Project + if: success() + run: ./mvnw --batch-mode install -D ci.build -D ci.release - name: Archive Native Image JAR if: success() uses: actions/upload-artifact@v2 with: - name: native-image + name: native-image-11 path: cli/target/*-native-image.jar - name: Archive Bash Completion Script if: success() uses: actions/upload-artifact@v2 with: - name: bash-completion-script + name: bash-completion-script-11 path: cli/target/sourcehawk-bash-completion.sh - name: Archive Manpages if: success() uses: actions/upload-artifact@v2 with: - name: manpages + name: manpages-11 path: gh-pages/manpages/sourcehawk*.1 - name: Aggregate Coverage Reports id: aggregate_coverage_reports @@ -106,10 +128,19 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ format('v{0}', steps.set_maven_project_version.outputs.RELEASE_VERSION) }} - release_name: ${{ format('{0} {1}', github.event.repository.name, steps.set_maven_project_version.outputs.RELEASE_VERSION) }} + release_name: ${{ format('{0} {1}', github.event.repository.name, needs.build.outputs.RELEASE_VERSION) }} body_path: CHANGELOG.md draft: ${{ github.event.inputs.draft }} prerelease: ${{ github.event.inputs.prerelease }} + - name: Publish Github Pages + if: success() + continue-on-error: true + uses: jamesives/github-pages-deploy-action@3.7.1 + with: + COMMIT_MESSAGE: ${{ format('Publishing github pages for release version {0}', needs.build.outputs.RELEASE_VERSION) }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages + FOLDER: gh-pages - name: Upload Sourcehawk Linux Executable if: success() continue-on-error: true @@ -119,7 +150,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/linux/target/sourcehawk - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-linux-x86_64 + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-linux-x86_64 asset_content_type: application/octet-stream - name: Upload Sourcehawk Debian Buster Package if: success() @@ -130,7 +161,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/debian/target/sourcehawk-debian-buster.deb - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-debian-buster-amd64.deb + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-debian-buster-amd64.deb asset_content_type: application/octet-stream - name: Upload Sourcehawk Ubuntu Focal Package if: success() @@ -141,7 +172,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/debian/target/sourcehawk-ubuntu-focal.deb - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-ubuntu-focal-amd64.deb + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-ubuntu-focal-amd64.deb asset_content_type: application/octet-stream - name: Upload Sourcehawk Centos 7 RPM Package if: success() @@ -152,7 +183,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/rpm/target/sourcehawk-centos-7.rpm - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.el7.x86_64.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.el7.x86_64.rpm asset_content_type: application/octet-stream - name: Upload Sourcehawk Centos 8 RPM Package if: success() @@ -163,7 +194,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/rpm/target/sourcehawk-centos-8.rpm - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.el8.x86_64.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.el8.x86_64.rpm asset_content_type: application/octet-stream - name: Upload Sourcehawk Fedora 33 RPM Package if: success() @@ -174,7 +205,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/rpm/target/sourcehawk-fedora-33.rpm - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.fc33.x86_64.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.fc33.x86_64.rpm asset_content_type: application/octet-stream - name: Upload Sourcehawk Fedora 34 RPM Package if: success() @@ -185,7 +216,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/rpm/target/sourcehawk-fedora-34.rpm - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.fc34.x86_64.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.fc34.x86_64.rpm asset_content_type: application/octet-stream - name: Upload Sourcehawk Fedora 35 RPM Package if: success() @@ -196,45 +227,8 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ./distributions/rpm/target/sourcehawk-fedora-35.rpm - asset_name: sourcehawk-${{ steps.set_maven_project_version.outputs.RELEASE_VERSION }}-1.fc35.x86_64.rpm + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-1.fc35.x86_64.rpm asset_content_type: application/octet-stream - - name: Publish Github Pages - if: success() - continue-on-error: true - uses: jamesives/github-pages-deploy-action@3.7.1 - with: - COMMIT_MESSAGE: ${{ format('Publishing github pages for release version {0}', steps.set_maven_project_version.outputs.RELEASE_VERSION) }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages - FOLDER: gh-pages - build-java11: - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.inputs.ref }} - - name: Setup Java and Maven - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Set Maven Project Version - shell: bash - run: | - RELEASE_VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | tail -1 | tr -d '\r\n') - [[ "$RELEASE_VERSION" == *"-SNAPSHOT" ]] && RELEASE_VERSION="${RELEASE_VERSION%"-SNAPSHOT"}" && ./mvnw --batch-mode versions:set -D removeSnapshot || true - [[ "$(git tag -l $RELEASE_VERSION)" == "$RELEASE_VERSION" ]] && echo "Tag $RELEASE_VERSION already exists" && exit 1 - echo ::set-output name=RELEASE_VERSION::$RELEASE_VERSION - - name: Build Maven Project - if: success() - run: ./mvnw --batch-mode install -D ci.build -D ci.release - - name: Archive Native Image JAR - if: success() - uses: actions/upload-artifact@v2 - with: - name: native-image-java11 - path: cli/target/*-native-image.jar build-mac-native-image: runs-on: macos-latest needs: build-java11 @@ -284,9 +278,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build-java11.outputs.RELEASE_ASSET_UPLOAD_URL }} + upload_url: ${{ needs.build.outputs.RELEASE_ASSET_UPLOAD_URL }} asset_path: build/sourcehawk - asset_name: sourcehawk-${{ needs.build-java11.outputs.RELEASE_VERSION }}-darwin-x86_64 + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-darwin-x86_64 asset_content_type: application/octet-stream - name: Upload Sourcehawk Mac Homebrew Tap Formula Archive id: upload_homebrew_tap_formula_archive @@ -296,20 +290,20 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build-java11.outputs.RELEASE_ASSET_UPLOAD_URL }} + upload_url: ${{ needs.build.outputs.RELEASE_ASSET_UPLOAD_URL }} asset_path: build/sourcehawk-homebrew-tap-formula.tar.gz - asset_name: sourcehawk-${{ needs.build-java11.outputs.RELEASE_VERSION }}-darwin-x86_64.tar.gz + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-darwin-x86_64.tar.gz asset_content_type: application/octet-stream - name: Update Optum Homebrew Tap Formula uses: mislav/bump-homebrew-formula-action@v1.10 continue-on-error: true with: formula-name: sourcehawk - tag-name: ${{ needs.build-java11.outputs.RELEASE_VERSION }} + tag-name: ${{ needs.build.outputs.RELEASE_VERSION }} homebrew-tap: optum/homebrew-tap base-branch: main download-url: ${{ steps.upload_homebrew_tap_formula_archive.outputs.browser_download_url }} - commit-message: Updating sourcehawk formula to latest release version ${{ needs.build-java11.outputs.RELEASE_VERSION }} + commit-message: Updating sourcehawk formula to latest release version ${{ needs.build.outputs.RELEASE_VERSION }} env: COMMITTER_TOKEN: ${{ secrets.GIT_COMMITTER_TOKEN }} build-windows-native-image: @@ -344,7 +338,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build-java11.outputs.RELEASE_ASSET_UPLOAD_URL }} + upload_url: ${{ needs.build.outputs.RELEASE_ASSET_UPLOAD_URL }} asset_path: sourcehawk.exe - asset_name: sourcehawk-${{ needs.build-java11.outputs.RELEASE_VERSION }}-windows-x86_64.exe + asset_name: sourcehawk-${{ needs.build.outputs.RELEASE_VERSION }}-windows-x86_64.exe asset_content_type: application/octet-stream diff --git a/distributions/linux/pom.xml b/distributions/linux/pom.xml index 63fff97..7a7159d 100644 --- a/distributions/linux/pom.xml +++ b/distributions/linux/pom.xml @@ -23,7 +23,6 @@ /home/${global.project.name} ${docker.workdir}/${global.project.name} ${project.build.directory}/${global.project.name} - true @@ -58,7 +57,6 @@ copy-resources - ${native.image.build.skip} ${project.parent.parent.basedir}/gh-pages/manpages @@ -84,7 +82,6 @@ copy-dependencies - ${native.image.build.skip} com.optum.sourcehawk sourcehawk-cli native-image @@ -98,7 +95,6 @@ copy - ${native.image.build.skip} com.optum.sourcehawk @@ -127,7 +123,6 @@ tag - ${native.image.build.skip} ${basedir}/native-image-builder/Dockerfile ${docker.image} ${project.version} @@ -157,7 +152,6 @@ exec - ${native.image.build.skip} ../scripts/extract-file-from-docker-container.sh ${docker.image}:${project.version} @@ -182,7 +176,6 @@ post-integration-test - ${native.image.build.skip} ${native.image} @@ -266,7 +259,6 @@ exec - ${native.image.build.skip} ${native.image} --version @@ -280,28 +272,6 @@ - - - java8 - - 1.8 - - - true - - - - - org.apache.maven.plugins - maven-failsafe-plugin - - true - - - - - - java11 @@ -310,7 +280,6 @@ 21.3.0-java11 - false @@ -322,7 +291,6 @@ 21.3.0-java17 - false diff --git a/distributions/pom.xml b/distributions/pom.xml index ebae0b5..a488bba 100644 --- a/distributions/pom.xml +++ b/distributions/pom.xml @@ -69,11 +69,4 @@ - - linux - debian - rpm - docker - - diff --git a/pom.xml b/pom.xml index 0aae6ea..ffedf93 100644 --- a/pom.xml +++ b/pom.xml @@ -213,6 +213,7 @@ distributions-build + !1.8 ci.build From df01c7f607bd88e35dc53f982a4ba9f86ce747e7 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 20:08:50 -0500 Subject: [PATCH 27/35] remote-repository: Fix dist pom --- distributions/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/distributions/pom.xml b/distributions/pom.xml index a488bba..ebae0b5 100644 --- a/distributions/pom.xml +++ b/distributions/pom.xml @@ -69,4 +69,11 @@ + + linux + debian + rpm + docker + + From 9597d7620da54b44a231257069f51bbee2ef4a26 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 20:30:23 -0500 Subject: [PATCH 28/35] remote-repository: Don't cache docker builders --- .../build-and-push-docker-builders.yml | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/.github/workflows/build-and-push-docker-builders.yml b/.github/workflows/build-and-push-docker-builders.yml index 8a62a1a..c9b2543 100644 --- a/.github/workflows/build-and-push-docker-builders.yml +++ b/.github/workflows/build-and-push-docker-builders.yml @@ -17,13 +17,6 @@ jobs: - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v1 - - name: Cache Docker Layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - name: Login to Github Container Registry uses: docker/login-action@v1 with: @@ -39,8 +32,6 @@ jobs: build-args: FROM_VERSION=java11-21.3.0 tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java11 push: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push graalvm-ce-21.3.0-java17 uses: docker/build-push-action@v2 with: @@ -50,8 +41,6 @@ jobs: build-args: FROM_VERSION=java17-21.3.0 tags: ghcr.io/optum/ci/nativeimage:graalvm-ce-21.3.0-java17 push: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild centos7 uses: docker/build-push-action@v2 with: @@ -61,8 +50,6 @@ jobs: build-args: FROM=centos:7 tags: ghcr.io/optum/ci/rpmbuild:centos7 push: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild centos8 uses: docker/build-push-action@v2 with: @@ -72,8 +59,6 @@ jobs: build-args: FROM=centos:8 tags: ghcr.io/optum/ci/rpmbuild:centos8 push: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild fedora33 uses: docker/build-push-action@v2 with: @@ -83,8 +68,6 @@ jobs: build-args: FROM=fedora:33 tags: ghcr.io/optum/ci/rpmbuild:fedora33 push: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild fedora34 uses: docker/build-push-action@v2 with: @@ -94,8 +77,6 @@ jobs: build-args: FROM=fedora:34 tags: ghcr.io/optum/ci/rpmbuild:fedora34 push: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Build and Push rpmbuild fedora35 uses: docker/build-push-action@v2 with: @@ -105,9 +86,3 @@ jobs: build-args: FROM=fedora:35 tags: ghcr.io/optum/ci/rpmbuild:fedora35 push: true - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - name: Relocate Docker Layer Cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache From 873a4cb2117917d355b72f1ec670fdfea6b11b1d Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sat, 22 Jan 2022 21:21:47 -0500 Subject: [PATCH 29/35] remote-repository: Only build dist for java 11, use 20.3.0 for windows native image --- .github/workflows/maven-ci.yml | 11 ++++++----- .github/workflows/release.yml | 2 +- pom.xml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index e061736..1f25c4e 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -79,26 +79,26 @@ jobs: SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} SONATYPE_GPG_PASSPHRASE: ${{ secrets.SONATYPE_GPG_PASSPHRASE }} - name: Archive Native Image JAR - if: success() + if: success() && (matrix.java == '11') uses: actions/upload-artifact@v2 with: name: native-image-${{ matrix.java }} path: cli/target/*-native-image.jar - name: Archive Bash Completion Script - if: success() + if: success() && (matrix.java == '11') uses: actions/upload-artifact@v2 with: name: bash-completion-script-${{ matrix.java }} path: cli/target/sourcehawk-bash-completion.sh - name: Archive Manpages - if: success() + if: success() && (matrix.java == '11') uses: actions/upload-artifact@v2 with: name: manpages-${{ matrix.java }} path: gh-pages/manpages/sourcehawk*.1 - name: Aggregate Coverage Reports id: aggregate_coverage_reports - if: success() + if: success() && (matrix.java == '11') run: echo ::set-output name=JACOCO_XML_REPORT_PATHS::$(find . -name "jacoco.xml" -printf '%P\n' | tr '\r\n' ',') - name: Analyze with SonarCloud if: success() && (github.event_name == 'push' && matrix.java == '11') @@ -170,6 +170,7 @@ jobs: build-windows-native-image: runs-on: windows-latest needs: build + continue-on-error: true steps: - uses: actions/download-artifact@v2 with: @@ -181,7 +182,7 @@ jobs: - name: Setup GraalVM uses: graalvm/setup-graalvm@v1 with: - version: '21.3.0' + version: '20.3.0' java-version: '11' components: 'native-image' github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a5ea1fd..9ffa6ea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -320,7 +320,7 @@ jobs: - name: Setup GraalVM uses: graalvm/setup-graalvm@v1 with: - version: '21.3.0' + version: '20.3.0' java-version: '11' components: 'native-image' github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/pom.xml b/pom.xml index ffedf93..39b2b17 100644 --- a/pom.xml +++ b/pom.xml @@ -213,7 +213,7 @@ distributions-build - !1.8 + 11 ci.build From e1c89f230724b88ef2785ab4274b997e928fcdc3 Mon Sep 17 00:00:00 2001 From: Brian Wyka <46069296+brianwyka@users.noreply.github.com> Date: Sat, 22 Jan 2022 23:35:55 -0500 Subject: [PATCH 30/35] Revert to Test Report --- .github/workflows/maven-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index 1f25c4e..c1a05b7 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -114,7 +114,6 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} report_paths: '**/target/surefire-reports/TEST-*.xml,**/target/failsafe-reports/TEST-*.xml' - check_name: Test Report (${{ matrix.java }}) build-mac-native-image: runs-on: macos-latest needs: build @@ -199,4 +198,4 @@ jobs: - name: Smoke Test if: success() shell: cmd - run: sourcehawk.exe help \ No newline at end of file + run: sourcehawk.exe help From 9068a86f830b39d3d7008137fea861167922b9b6 Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sun, 23 Jan 2022 19:07:55 -0500 Subject: [PATCH 31/35] remote-repository: Remove git-commitid-plugin, fixes #68 --- .github/workflows/maven-ci.yml | 14 +++++++------- .github/workflows/release.yml | 13 ++++++++----- distributions/debian/pom.xml | 32 ++++++++++++++++++++++---------- distributions/pom.xml | 11 ----------- distributions/rpm/pom.xml | 26 +++++++++++++++++++++++--- 5 files changed, 60 insertions(+), 36 deletions(-) diff --git a/.github/workflows/maven-ci.yml b/.github/workflows/maven-ci.yml index c1a05b7..cbd46ff 100644 --- a/.github/workflows/maven-ci.yml +++ b/.github/workflows/maven-ci.yml @@ -82,19 +82,19 @@ jobs: if: success() && (matrix.java == '11') uses: actions/upload-artifact@v2 with: - name: native-image-${{ matrix.java }} + name: native-image path: cli/target/*-native-image.jar - name: Archive Bash Completion Script if: success() && (matrix.java == '11') uses: actions/upload-artifact@v2 with: - name: bash-completion-script-${{ matrix.java }} + name: bash-completion-script path: cli/target/sourcehawk-bash-completion.sh - name: Archive Manpages if: success() && (matrix.java == '11') uses: actions/upload-artifact@v2 with: - name: manpages-${{ matrix.java }} + name: manpages path: gh-pages/manpages/sourcehawk*.1 - name: Aggregate Coverage Reports id: aggregate_coverage_reports @@ -120,15 +120,15 @@ jobs: steps: - uses: actions/download-artifact@v2 with: - name: native-image-11 + name: native-image path: build - uses: actions/download-artifact@v2 with: - name: bash-completion-script-11 + name: bash-completion-script path: build - uses: actions/download-artifact@v2 with: - name: manpages-11 + name: manpages path: build - name: Rename Native Image JAR working-directory: build @@ -173,7 +173,7 @@ jobs: steps: - uses: actions/download-artifact@v2 with: - name: native-image-11 + name: native-image path: build - name: Rename Native Image JAR working-directory: build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ffa6ea..3d86f5d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -82,19 +82,19 @@ jobs: if: success() uses: actions/upload-artifact@v2 with: - name: native-image-11 + name: native-image path: cli/target/*-native-image.jar - name: Archive Bash Completion Script if: success() uses: actions/upload-artifact@v2 with: - name: bash-completion-script-11 + name: bash-completion-script path: cli/target/sourcehawk-bash-completion.sh - name: Archive Manpages if: success() uses: actions/upload-artifact@v2 with: - name: manpages-11 + name: manpages path: gh-pages/manpages/sourcehawk*.1 - name: Aggregate Coverage Reports id: aggregate_coverage_reports @@ -235,7 +235,7 @@ jobs: steps: - uses: actions/download-artifact@v2 with: - name: native-image-java11 + name: native-image path: build - uses: actions/download-artifact@v2 with: @@ -303,7 +303,10 @@ jobs: homebrew-tap: optum/homebrew-tap base-branch: main download-url: ${{ steps.upload_homebrew_tap_formula_archive.outputs.browser_download_url }} - commit-message: Updating sourcehawk formula to latest release version ${{ needs.build.outputs.RELEASE_VERSION }} + commit-message: | + Sourcehawk ${{ needs.build.outputs.RELEASE_VERSION }} + + Updating sourcehawk formula to latest release version env: COMMITTER_TOKEN: ${{ secrets.GIT_COMMITTER_TOKEN }} build-windows-native-image: diff --git a/distributions/debian/pom.xml b/distributions/debian/pom.xml index 158df2d..a9116e7 100644 --- a/distributions/debian/pom.xml +++ b/distributions/debian/pom.xml @@ -21,7 +21,7 @@ ${project.build.directory}/debian ${bintray.package} ${project.version} - ~${git.commit.time}+${git.commit.id.abbrev} + amd64 @@ -49,15 +49,27 @@ - pl.project13.maven - git-commit-id-plugin - - ${maven.build.timestamp.format} - ${maven.multiModuleProjectDirectory}/.git - false - true - ${project.build.directory}/git.properties - + org.codehaus.gmavenplus + gmavenplus-plugin + + + save-git-information + validate + + execute + + + + + + + + diff --git a/distributions/pom.xml b/distributions/pom.xml index ebae0b5..71a430b 100644 --- a/distributions/pom.xml +++ b/distributions/pom.xml @@ -36,17 +36,6 @@ - - pl.project13.maven - git-commit-id-plugin - - ${maven.build.timestamp.format} - ${maven.multiModuleProjectDirectory}/.git - false - true - ${project.build.directory}/git.properties - - maven-enforcer-plugin diff --git a/distributions/rpm/pom.xml b/distributions/rpm/pom.xml index 19e90da..aaf7454 100644 --- a/distributions/rpm/pom.xml +++ b/distributions/rpm/pom.xml @@ -20,9 +20,10 @@ ${bintray.package} ${project.version} - a.${git.commit.time}~${git.commit.id.abbrev} + x86_64 ${project.build.directory}/rpmbuild + @@ -51,8 +52,27 @@ - pl.project13.maven - git-commit-id-plugin + org.codehaus.gmavenplus + gmavenplus-plugin + + + save-git-information + validate + + execute + + + + + + + + From 9d27746191c1293905c3bac892a69b4f1f590f8c Mon Sep 17 00:00:00 2001 From: Brian Wyka Date: Sun, 23 Jan 2022 19:23:09 -0500 Subject: [PATCH 32/35] remote-repository: Fix git scripts format --- distributions/debian/pom.xml | 2 +- distributions/rpm/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distributions/debian/pom.xml b/distributions/debian/pom.xml index a9116e7..0d64622 100644 --- a/distributions/debian/pom.xml +++ b/distributions/debian/pom.xml @@ -61,7 +61,7 @@