[hailctl] Upgrade dataproc to image version 2.2.64 (#15077)

Also update the dataproc requirements to match the version they are
actually running.

## Security Assessment
- This change cannot impact the Hail Batch instance as deployed by Broad
Institute in GCP

Author: chrisvittal

ci/pinned-requirements.txt

@@ -6,7 +6,7 @@ certifi==2025.8.3
     #   -c ci/../hail/python/dev/pinned-requirements.txt
     #   -c ci/../hail/python/pinned-requirements.txt
     #   requests
-cffi==1.17.1
+cffi==2.0.0
     # via
     #   -c ci/../hail/python/dev/pinned-requirements.txt
     #   -c ci/../hail/python/pinned-requirements.txt
@@ -22,7 +22,7 @@ click==8.2.1
     #   -c ci/../hail/python/dev/pinned-requirements.txt
     #   -c ci/../hail/python/pinned-requirements.txt
     #   zulip
-cryptography==45.0.6
+cryptography==45.0.7
     # via
     #   -c ci/../hail/python/pinned-requirements.txt
     #   pyjwt
@@ -37,7 +37,7 @@ idna==3.10
     #   -c ci/../hail/python/pinned-requirements.txt
     #   -c ci/../web_common/pinned-requirements.txt
     #   requests
-pycparser==2.22
+pycparser==2.23
     # via
     #   -c ci/../hail/python/dev/pinned-requirements.txt
     #   -c ci/../hail/python/pinned-requirements.txt

gear/pinned-requirements.txt

@@ -62,7 +62,7 @@ frozenlist==1.7.0
     #   aiosignal
 google-api-core==2.25.1
     # via google-api-python-client
-google-api-python-client==2.179.0
+google-api-python-client==2.181.0
     # via google-cloud-profiler
 google-auth==2.40.3
     # via
@@ -80,7 +80,7 @@ google-cloud-profiler==4.1.0
     # via -r gear/requirements.txt
 googleapis-common-protos==1.70.0
     # via google-api-core
-httplib2==0.22.0
+httplib2==0.31.0
     # via
     #   google-api-python-client
     #   google-auth-httplib2
@@ -116,7 +116,7 @@ propcache==0.3.2
     #   yarl
 proto-plus==1.26.1
     # via google-api-core
-protobuf==6.32.0
+protobuf==6.32.1
     # via
     #   google-api-core
     #   google-cloud-profiler
@@ -137,7 +137,7 @@ pymysql==1.1.2
     # via
     #   -r gear/requirements.txt
     #   aiomysql
-pyparsing==3.2.3
+pyparsing==3.2.4
     # via
     #   -c gear/../hail/python/dev/pinned-requirements.txt
     #   httplib2

hail/python/dataproc-pre-installed-requirements.txt

@@ -1,4 +1,4 @@
 # https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-release-2.2
 #
-# 2.2.5-debian12
-pyspark==3.5.0
+# 2.2.64-debian12
+pyspark==3.5.3

hail/python/dev/pinned-requirements.txt

@@ -64,7 +64,7 @@ certifi==2025.8.3
     #   httpcore
     #   httpx
     #   requests
-cffi==1.17.1
+cffi==2.0.0
     # via
     #   -c hail/python/dev/../pinned-requirements.txt
     #   argon2-cffi-bindings
@@ -122,15 +122,15 @@ exceptiongroup==1.3.0
     #   pytest
 execnet==2.1.1
     # via pytest-xdist
-executing==2.2.0
+executing==2.2.1
     # via
     #   devtools
     #   stack-data
 fastjsonschema==2.21.2
     # via nbformat
 filelock==3.19.1
     # via virtualenv
-fonttools==4.59.1
+fonttools==4.59.2
     # via matplotlib
 fqdn==1.5.1
     # via jsonschema
@@ -147,7 +147,7 @@ httpcore==1.0.9
     # via httpx
 httpx==0.28.1
     # via jupyterlab
-identify==2.6.13
+identify==2.6.14
     # via pre-commit
 idna==3.10
     # via
@@ -206,7 +206,7 @@ jsonschema==4.25.1
     #   jupyter-events
     #   jupyterlab-server
     #   nbformat
-jsonschema-specifications==2025.4.1
+jsonschema-specifications==2025.9.1
     # via jsonschema
 jupyter==1.1.1
     # via -r hail/python/dev/requirements.txt
@@ -232,7 +232,7 @@ jupyter-core==5.8.1
     #   notebook
 jupyter-events==0.12.0
     # via jupyter-server
-jupyter-lsp==2.2.6
+jupyter-lsp==2.3.0
     # via jupyterlab
 jupyter-server==2.17.0
     # via
@@ -242,7 +242,7 @@ jupyter-server==2.17.0
     #   notebook-shim
 jupyter-server-terminals==0.5.3
     # via jupyter-server
-jupyterlab==4.4.6
+jupyterlab==4.4.7
     # via jupyter
 jupyterlab-pygments==0.3.0
     # via nbconvert
@@ -259,15 +259,15 @@ markupsafe==3.0.2
     #   -c hail/python/dev/../pinned-requirements.txt
     #   jinja2
     #   nbconvert
-matplotlib==3.10.5
+matplotlib==3.10.6
     # via -r hail/python/dev/requirements.txt
 matplotlib-inline==0.1.7
     # via
     #   ipykernel
     #   ipython
 mccabe==0.7.0
     # via pylint
-mistune==3.1.3
+mistune==3.1.4
     # via nbconvert
 mock==5.1.0
     # via -r hail/python/dev/requirements.txt
@@ -276,7 +276,7 @@ multidict==6.6.4
     #   -c hail/python/dev/../pinned-requirements.txt
     #   aiohttp
     #   yarl
-nbclassic==1.3.1
+nbclassic==1.3.2
     # via notebook
 nbclient==0.10.2
     # via nbconvert
@@ -307,7 +307,7 @@ nodeenv==1.9.1
     # via
     #   pre-commit
     #   pyright
-nodejs-wheel-binaries==22.18.0
+nodejs-wheel-binaries==22.19.0
     # via pyright
 notebook==6.5.7
     # via
@@ -352,7 +352,7 @@ pillow==11.3.0
     #   -c hail/python/dev/../pinned-requirements.txt
     #   -r hail/python/dev/requirements.txt
     #   matplotlib
-platformdirs==4.3.8
+platformdirs==4.4.0
     # via
     #   jupyter-core
     #   pylint
@@ -365,7 +365,7 @@ prometheus-client==0.22.1
     # via
     #   jupyter-server
     #   notebook
-prompt-toolkit==3.0.51
+prompt-toolkit==3.0.52
     # via
     #   ipython
     #   jupyter-console
@@ -384,7 +384,7 @@ pure-eval==0.2.3
     # via stack-data
 py==1.11.0
     # via pytest-forked
-pycparser==2.22
+pycparser==2.23
     # via
     #   -c hail/python/dev/../pinned-requirements.txt
     #   cffi
@@ -399,11 +399,11 @@ pygments==2.19.2
     #   sphinx
 pylint==3.3.8
     # via -r hail/python/dev/requirements.txt
-pyparsing==3.2.3
+pyparsing==3.2.4
     # via matplotlib
 pyproject-hooks==1.2.0
     # via build
-pyright==1.1.404
+pyright==1.1.405
     # via -r hail/python/dev/requirements.txt
 pytest==7.4.4
     # via
@@ -426,7 +426,7 @@ pytest-instafail==0.5.0
     # via -r hail/python/dev/requirements.txt
 pytest-metadata==3.1.1
     # via pytest-html
-pytest-mock==3.14.1
+pytest-mock==3.15.0
     # via -r hail/python/dev/requirements.txt
 pytest-timeout==2.4.0
     # via -r hail/python/dev/requirements.txt
@@ -449,7 +449,7 @@ pyyaml==6.0.2
     #   -c hail/python/dev/../pinned-requirements.txt
     #   jupyter-events
     #   pre-commit
-pyzmq==27.0.2
+pyzmq==27.1.0
     # via
     #   ipykernel
     #   jupyter-client
@@ -476,7 +476,7 @@ rfc3986-validator==0.1.1
     #   jupyter-events
 rfc3987-syntax==1.1.0
     # via jsonschema
-rpds-py==0.27.0
+rpds-py==0.27.1
     # via
     #   jsonschema
     #   referencing
@@ -498,7 +498,7 @@ sniffio==1.3.1
     # via anyio
 snowballstemmer==3.0.1
     # via sphinx
-soupsieve==2.7
+soupsieve==2.8
     # via beautifulsoup4
 sphinx==6.2.1
     # via
@@ -579,13 +579,13 @@ types-decorator==5.2.0.20250324
     # via -r hail/python/dev/requirements.txt
 types-deprecated==1.2.15.20250304
     # via -r hail/python/dev/requirements.txt
-types-pymysql==1.1.0.20250822
+types-pymysql==1.1.0.20250914
     # via -r hail/python/dev/requirements.txt
 types-python-dateutil==2.9.0.20250822
     # via
     #   -r hail/python/dev/requirements.txt
     #   arrow
-types-pyyaml==6.0.12.20250822
+types-pyyaml==6.0.12.20250915
     # via -r hail/python/dev/requirements.txt
 types-requests==2.31.0.6
     # via -r hail/python/dev/requirements.txt
@@ -620,7 +620,7 @@ urllib3==2.5.0
     # via
     #   -c hail/python/dev/../pinned-requirements.txt
     #   requests
-uv==0.8.13
+uv==0.8.17
     # via -r hail/python/dev/requirements.txt
 uv-build==0.8.13
     # via -r hail/python/dev/requirements.txt

hail/python/hailtop/hailctl/dataproc/start.py

@@ -129,7 +129,7 @@ class VepVersion(str, Enum):
 
 ANNOTATION_DB_BUCKETS = ["hail-datasets-us-central1", "hail-datasets-europe-west1"]
 
-IMAGE_VERSION = '2.2.60-debian12'
+IMAGE_VERSION = '2.2.64-debian12'
 
 
 def start(

hail/python/hailtop/pinned-requirements.txt

@@ -14,23 +14,23 @@ attrs==25.3.0
     # via aiohttp
 azure-common==1.1.28
     # via azure-mgmt-storage
-azure-core==1.35.0
+azure-core==1.35.1
     # via
     #   azure-identity
     #   azure-mgmt-core
     #   azure-storage-blob
     #   msrest
-azure-identity==1.24.0
+azure-identity==1.25.0
     # via -r hail/python/hailtop/requirements.txt
 azure-mgmt-core==1.6.0
     # via azure-mgmt-storage
 azure-mgmt-storage==20.1.0
     # via -r hail/python/hailtop/requirements.txt
 azure-storage-blob==12.26.0
     # via -r hail/python/hailtop/requirements.txt
-boto3==1.40.16
+boto3==1.40.30
     # via -r hail/python/hailtop/requirements.txt
-botocore==1.40.16
+botocore==1.40.30
     # via
     #   -r hail/python/hailtop/requirements.txt
     #   boto3
@@ -41,7 +41,7 @@ certifi==2025.8.3
     # via
     #   msrest
     #   requests
-cffi==1.17.1
+cffi==2.0.0
     # via
     #   cryptography
     #   pycares
@@ -51,7 +51,7 @@ click==8.2.1
     # via typer
 commonmark==0.9.1
     # via rich
-cryptography==45.0.6
+cryptography==45.0.7
     # via
     #   azure-identity
     #   azure-storage-blob
@@ -104,7 +104,7 @@ nest-asyncio==1.6.0
     # via -r hail/python/hailtop/requirements.txt
 oauthlib==3.3.1
     # via requests-oauthlib
-orjson==3.11.2
+orjson==3.11.3
     # via -r hail/python/hailtop/requirements.txt
 propcache==0.3.2
     # via
@@ -116,9 +116,9 @@ pyasn1==0.6.1
     #   rsa
 pyasn1-modules==0.4.2
     # via google-auth
-pycares==4.10.0
+pycares==4.11.0
     # via aiodns
-pycparser==2.22
+pycparser==2.23
     # via cffi
 pygments==2.19.2
     # via rich
@@ -146,7 +146,7 @@ rich==12.6.0
     #   typer
 rsa==4.9.1
     # via google-auth
-s3transfer==0.13.1
+s3transfer==0.14.0
     # via boto3
 shellingham==1.5.4
     # via typer
@@ -159,7 +159,7 @@ sortedcontainers==2.4.0
     # via -r hail/python/hailtop/requirements.txt
 tabulate==0.9.0
     # via -r hail/python/hailtop/requirements.txt
-typer==0.16.1
+typer==0.17.4
     # via -r hail/python/hailtop/requirements.txt
 typing-extensions==4.15.0
     # via