Vs 1736 - changing import to use parquet

.dockstore.yml

@@ -211,17 +211,32 @@ workflows:
          branches:
              - master
              - ah_var_store
+             - VS-1736
+             - vs_1800_prep_for_merge
          tags:
              - /.*/
-   - name: GvsBulkIngestGenomes
+   - name: BulkIngestWriteAPI
      subclass: WDL
      primaryDescriptorPath: /scripts/variantstore/wdl/GvsBulkIngestGenomes.wdl
      filters:
          branches:
              - master
              - ah_var_store
+             - vs_1799_fix_parquet_exome_npe
+             - vs_1809_parquet_ploidy
          tags:
              - /.*/
+   - name: BulkIngestParquet
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsBulkIngestGenomes.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - vs_1799_fix_parquet_exome_npe
+         - vs_1809_parquet_ploidy
+       tags:
+         - /.*/
    - name: GvsPrepareRangesCallset
      subclass: WDL
      primaryDescriptorPath: /scripts/variantstore/wdl/GvsPrepareRangesCallset.wdl
@@ -343,9 +358,39 @@ workflows:
          branches:
              - master
              - ah_var_store
-             - vs_1730_clusters_leaking
+             - vs_1800_prep_for_merge
          tags:
              - /.*/
+   - name: GvsQuickstartIntegrationParquet1809
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsQuickstartIntegration.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - vs_1809_parquet_ploidy
+       tags:
+         - /.*/
+   - name: GvsQuickstartIntegrationWriteAPI1809
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsQuickstartIntegration.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - vs_1809_parquet_ploidy
+       tags:
+         - /.*/
+   - name: GvsQuickstartIntegrationBGEOnlyParquet1809
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsQuickstartIntegration.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - vs_1809_parquet_ploidy
+       tags:
+         - /.*/
    - name: GvsIngestTieout
      subclass: WDL
      primaryDescriptorPath: /scripts/variantstore/wdl/test/GvsIngestTieout.wdl
@@ -462,6 +507,14 @@ workflows:
        branches:
          - master
          - ah_var_store
+   - name: GvsMapUnmappedVIDs
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/variant-annotations-table/GvsMapUnmappedVIDs.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - vs_1757_dropped_duplicates
    - name: MergePgenHierarchicalWdl
      subclass: WDL
      primaryDescriptorPath: /scripts/variantstore/wdl/MergePgenHierarchical.wdl
@@ -481,3 +534,30 @@ workflows:
      filters:
        branches:
          - ah_var_store
+   - name: SearchGVCFsAtSite
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/variant-annotations-table/SearchGVCFsAtSite.wdl
+     filters:
+       branches:
+         - ah_var_store
+         - dst_2716_an_divergence_echo_foxtrot
+   - name: GvsCreateParticipantMappingTable
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/variant-annotations-table/GvsCreateParticipantMappingTable.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - vs_1757_dropped_duplicates
+       tags:
+         - /.*/
+   - name: GvsMapDroppedDuplicateVIDs
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/variant-annotations-table/GvsMapDroppedDuplicateVIDs.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - vs_1757_dropped_duplicates
+       tags:
+         - /.*/

.github/workflows/gatk-tests.yml

@@ -104,7 +104,7 @@ jobs:
       #Google Cloud stuff
       - id: 'gcloud-auth'
         if: needs.check-secrets.outputs.google-credentials == 'true'
-        uses: google-github-actions/auth@v0
+        uses: google-github-actions/auth@v3
         with:
           credentials_json: ${{ secrets.GCP_CREDENTIALS }}
           project_id: ${{ env.HELLBENDER_TEST_PROJECT }}
@@ -175,7 +175,7 @@ jobs:
 
       #Google Cloud stuff
       - id: 'gcloud-auth'
-        uses: google-github-actions/auth@v0
+        uses: google-github-actions/auth@v3
         if: needs.check-secrets.outputs.google-credentials == 'true'
         with:
           credentials_json: ${{ secrets.GCP_CREDENTIALS }}

build.gradle

@@ -74,7 +74,7 @@ final guavaVersion = System.getProperty('guava.version', '32.1.3-jre')
 final log4j2Version = System.getProperty('log4j2Version', '2.24.1')
 final testNGVersion = System.getProperty('testNGVersion', '7.7.0')
 final googleCloudNioVersion = System.getProperty('googleCloudNioVersion','0.127.8')
-final gklVersion = System.getProperty('gklVersion', '0.9.0')
+final gklVersion = System.getProperty('gklVersion', '0.9.1')
 
 final baseJarName = 'gatk'
 final secondaryBaseJarName = 'hellbender'
@@ -389,6 +389,13 @@ dependencies {
     // pgen jni
     implementation('org.broadinstitute:pgenjni:1.0.1')
 
+    // parquet writing
+    implementation('org.apache.parquet:parquet-common:1.13.1')
+    implementation('org.apache.parquet:parquet-encoding:1.13.1')
+    implementation('org.apache.parquet:parquet-column:1.13.1')
+    implementation('org.apache.parquet:parquet-hadoop:1.13.1')
+    implementation 'org.apache.parquet:parquet-avro:1.13.1'
+
     testUtilsImplementation sourceSets.main.output
     testUtilsImplementation 'org.testng:testng:' + testNGVersion
     testUtilsImplementation 'org.apache.hadoop:hadoop-minicluster:' + hadoopVersion

build.log

@@ -0,0 +1,14 @@
+Option -e requires an argument.
+ Usage: ./build_docker.sh: -e <GITHUB_TAG> [-psl] 
+ where <GITHUB_TAG> is the github tag (or hash when -s is used) to use in building the docker image
+ (e.g. bash build_docker.sh -e 1.0.0.0-alpha1.2.1)
+ Optional arguments:  
+ -s 	 The GITHUB_TAG (-e parameter) is actually a github hash, not tag.  git hashes cannot be pushed as latest, so -l is implied.  
+ -l 	 Do not also push the image to the 'latest' tag. 
+ -u 	 Do not run the unit tests. 
+ -m 	 Build the lite image (which does not contain the conda environment). 
+ -d <STAGING_DIR> 	 staging directory to grab code from repo and build the docker image.  If unspecified, then use whatever is in current dir (do not go to the repo).  NEVER SPECIFY YOUR WORKING DIR 
+ -p 	 (GATK4 developers only) push image to docker hub once complete.  This will use the GITHUB_TAG in dockerhub as well. 
+ 		 Unless -l is specified, this will also push this image to the 'latest' tag. 
+ -r 	 (GATK4 developers only) Do not remove the unit test docker container.  This is useful for debugging failing unit tests. 
+ -t <PULL_REQUEST_NUMBER>	 (Travis CI only) The pull request number.  This is only used during pull request builds on Travis CI. 

docs/gvs_ingest_refactoring.md

@@ -0,0 +1,87 @@
+# GVS Ingest Package Refactoring
+
+## Overview
+
+The `org.broadinstitute.hellbender.tools.gvs.ingest` package has been refactored to support multiple output formats through a consistent interface-based architecture. This refactoring separates concerns by format type and introduces polymorphism for easier addition of new output formats.
+
+## Package Structure
+
+```
+src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/
+├── VetWriter.java                    (interface)
+├── RefRangesWriter.java              (interface)
+├── SamplePloidyWriter.java           (interface)
+├── avro/
+│   └── RefRangesAvroWriter.java
+├── bq/
+│   ├── AbstractBQWriter.java         (abstract base)
+│   ├── VetBQWriter.java
+│   ├── RefRangesBQWriter.java
+│   └── SamplePloidyBQWriter.java
+├── parquet/
+│   ├── AbstractParquetFileWriter.java (abstract base)
+│   ├── VetParquetFileWriter.java
+│   ├── RefRangesParquetFileWriter.java
+│   ├── SamplePloidyParquetFileWriter.java
+│   └── HeaderParquetFileWriter.java
+└── tsv/
+    ├── VetTsvWriter.java
+    └── RefRangesTsvWriter.java
+```
+
+## Key Design Elements
+
+### Writer Interfaces
+
+Three core interfaces define the write operations for GVS data types:
+
+- **`VetWriter`** - Writes variant information (location, sample ID, variant context)
+- **`RefRangesWriter`** - Writes reference ranges (location, length, state, compressed data)
+- **`SamplePloidyWriter`** - Writes chromosome ploidy information
+
+All interfaces extend `Closeable` and include a default `commitData()` no-op method that implementations can override.
+
+### Abstract Base Classes
+
+Two abstract base classes provide shared infrastructure for their respective output formats:
+
+- **`AbstractBQWriter`** - Handles BigQuery Write API operations via `PendingBQWriter`
+  - Provides `write(JSONObject)` for adding rows
+  - Implements `commitData()` to flush and commit write streams
+
+- **`AbstractParquetFileWriter`** - Handles Parquet file operations
+  - Uses composition pattern with `ParquetWriteSupport`
+  - Provides `write(JSONObject)` for writing records
+  - Manages compression and schema configuration
+
+### Format-Specific Implementations
+
+#### BigQuery (`bq/`)
+- `VetBQWriter`, `RefRangesBQWriter`, `SamplePloidyBQWriter`
+- Extend `AbstractBQWriter` and implement respective interfaces
+- Write data directly to BigQuery via the Write API
+
+#### Parquet (`parquet/`)
+- `VetParquetFileWriter`, `RefRangesParquetFileWriter`, `SamplePloidyParquetFileWriter`
+- Extend `AbstractParquetFileWriter` and implement respective interfaces
+- Write data to local Parquet files for later bulk loading
+
+#### TSV (`tsv/`)
+- `VetTsvWriter`, `RefRangesTsvWriter`
+- Direct implementations for tab-separated value output
+
+#### Avro (`avro/`)
+- `RefRangesAvroWriter`
+- Legacy format support
+
+## Benefits
+
+1. **Separation of Concerns**: Output format logic is isolated in dedicated packages
+2. **Extensibility**: New formats can be added by implementing the writer interfaces
+3. **Code Reuse**: Abstract base classes eliminate duplication across similar implementations
+4. **Type Safety**: Interface contracts ensure consistent APIs across formats
+5. **Testability**: Each implementation can be tested independently
+
+## Note
+
+VCF header writing does not yet have a dedicated interface; that should be added when VS-1803 (support writing VCF header data with Parquet) is implemented.