diff --git a/README.md b/README.md index cf18da22..bc74a17b 100644 --- a/README.md +++ b/README.md @@ -69,11 +69,11 @@ To run tests: `mvn test` To build binary: `mvn package` -- which will produce `tool/target/datacommons-import-tool-0.1-jar-with-dependencies.jar` +- which will produce `tool/target/datacommons-import-tool--jar-with-dependencies.jar` - and you can run it with ```bash - java -jar tool/target/datacommons-import-tool-0.1-jar-with-dependencies.jar + java -jar tool/target/datacommons-import-tool--jar-with-dependencies.jar ``` > To run the above maven commands on M1 macs ([details][m1]), use the `-Dos.arch=x86_64` option. @@ -92,11 +92,11 @@ To run tests: `mvn test` To build binary: `mvn package` -- which will produce `server/target/datacommons-server-0.1.jar` +- which will produce `server/target/datacommons-server-.jar` - and you can run it with ```bash - java -jar server/target/datacommons-server-0.1.jar + java -jar server/target/datacommons-server-.jar ``` Send a request: @@ -149,6 +149,14 @@ steps in [contributing.md](contributing.md). Wait for approval of the Pull Request and merge the change. +### Creating a Release + +1. Update the version in `pom.xml` to the release version (remove `-SNAPSHOT`). +2. Build the project: `mvn clean package`. +3. Create a new Release on GitHub. +4. Upload the built artifacts (e.g., `tool/target/datacommons-import-tool--jar-with-dependencies.jar`) to the GitHub release. +5. Update the version in `pom.xml` to the next snapshot version (e.g., `0.3.2-SNAPSHOT`) and raise a Pull Request. + ## License Apache 2.0 diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 46d36006..9cb6a77f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -6,7 +6,14 @@ steps: - AUTOPUSH_DC_API_KEY - name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', 'tool/target/datacommons-import-tool-0.1-SNAPSHOT-jar-with-dependencies.jar', 'gs://datacommons_public/import_tools/import-tool.jar'] + entrypoint: 'bash' + args: + - '-c' + - | + # Find the JAR file using wildcard + JAR_FILE=$(ls tool/target/datacommons-import-tool-*-jar-with-dependencies.jar | head -n 1) + echo "Found JAR: ${JAR_FILE}" + gsutil cp "${JAR_FILE}" "gs://datacommons_public/import_tools/import-tool.jar" - name: 'gcr.io/cloud-builders/gcloud' args: ['builds', 'triggers', 'run', 'dc-import-executor', '--branch=master', '--substitutions', '_DOCKER_IMAGE=us-docker.pkg.dev/datcom-ci/gcr.io/dc-import-executor'] diff --git a/pipeline/differ/template.sh b/pipeline/differ/template.sh index 169c87fa..084d9361 100644 --- a/pipeline/differ/template.sh +++ b/pipeline/differ/template.sh @@ -4,13 +4,17 @@ OPERATION=$1 if [ "$OPERATION" == "deploy" ]; then echo "Deploying Dataflow Flex Template..." + # Find the JAR file using wildcard + JAR_FILE=$(ls target/differ-bundled-*.jar | head -n 1) + echo "Found JAR: ${JAR_FILE}" + gcloud dataflow flex-template build \ "gs://datcom-templates/templates/flex/differ.json" \ --image-gcr-path "gcr.io/datcom-ci/dataflow-templates/differ:latest" \ --sdk-language "JAVA" \ --flex-template-base-image JAVA17 \ --metadata-file "metadata.json" \ - --jar "target/differ-bundled-0.1-SNAPSHOT.jar" \ + --jar "${JAR_FILE}" \ --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="org.datacommons.pipeline.differ.DifferPipeline" elif [ "$OPERATION" == "run" ]; then echo "Running Dataflow Flex Template..." diff --git a/pipeline/ingestion/cloudbuild.yaml b/pipeline/ingestion/cloudbuild.yaml index b51a56c5..86823123 100644 --- a/pipeline/ingestion/cloudbuild.yaml +++ b/pipeline/ingestion/cloudbuild.yaml @@ -19,24 +19,23 @@ steps: # 2. Build the Dataflow Flex Template # This step uses the built JAR to create the Flex Template image and spec file. - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' - entrypoint: 'gcloud' + entrypoint: 'bash' args: - - 'dataflow' - - 'flex-template' - - 'build' - - 'gs://$_TEMPLATE_BUCKET/templates/flex/ingestion.json' - - '--image-gcr-path' - - '${_IMAGE_GCR_PATH}:$SHORT_SHA' - - '--sdk-language' - - 'JAVA' - - '--flex-template-base-image' - - 'JAVA17' - - '--metadata-file' - - 'pipeline/ingestion/metadata.json' - - '--jar' - - 'pipeline/ingestion/target/ingestion-bundled-${_VERSION}.jar' - - '--env' - - 'FLEX_TEMPLATE_JAVA_MAIN_CLASS=org.datacommons.ingestion.pipeline.ImportGroupPipeline' + - '-c' + - | + # Find the JAR file using wildcard + # Note: We are in the root of the workspace context for the build. + JAR_FILE=$(ls pipeline/ingestion/target/ingestion-bundled-*.jar | head -n 1) + echo "Found JAR: ${JAR_FILE}" + + gcloud dataflow flex-template build \ + gs://$_TEMPLATE_BUCKET/templates/flex/ingestion.json \ + --image-gcr-path "${_IMAGE_GCR_PATH}:$SHORT_SHA" \ + --sdk-language JAVA \ + --flex-template-base-image JAVA17 \ + --metadata-file pipeline/ingestion/metadata.json \ + --jar "${JAR_FILE}" \ + --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=org.datacommons.ingestion.pipeline.ImportGroupPipeline id: 'build-flex-template' availableSecrets: @@ -45,6 +44,6 @@ availableSecrets: env: AUTOPUSH_DC_API_KEY substitutions: - _VERSION: "0.1-SNAPSHOT" + _TEMPLATE_BUCKET: "datcom-templates" _IMAGE_GCR_PATH: "gcr.io/datcom-ci/dataflow-templates/ingestion" diff --git a/pipeline/ingestion/template.sh b/pipeline/ingestion/template.sh index 6381249b..709b7e5a 100755 --- a/pipeline/ingestion/template.sh +++ b/pipeline/ingestion/template.sh @@ -4,13 +4,17 @@ OPERATION=$1 if [ "$OPERATION" == "deploy" ]; then echo "Deploying Dataflow Flex Template..." + # Find the JAR file using wildcard + JAR_FILE=$(ls target/ingestion-bundled-*.jar | head -n 1) + echo "Found JAR: ${JAR_FILE}" + gcloud dataflow flex-template build \ "gs://datcom-templates/templates/flex/ingestion.json" \ --image-gcr-path "gcr.io/datcom-ci/dataflow-templates/ingestion:latest" \ --sdk-language "JAVA" \ --flex-template-base-image JAVA17 \ --metadata-file "metadata.json" \ - --jar "target/ingestion-bundled-0.1-SNAPSHOT.jar" \ + --jar "${JAR_FILE}" \ --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="org.datacommons.ingestion.pipeline.ImportGroupPipeline" elif [ "$OPERATION" == "run" ]; then echo "Running Dataflow Flex Template..." diff --git a/pom.xml b/pom.xml index d508f96f..5acbf76b 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ 17 17 - 0.1-SNAPSHOT + 0.3.1-SNAPSHOT 2.67.0 2.10.1 1.7.1