diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 12a163dde4..93ea99fb6a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -34,7 +34,7 @@ body: description: What version of Heat are you running? options: - main (development branch) - - 1.6.x + - 1.7.x - other validations: required: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index af57812b6c..55fa7882c6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,10 +27,10 @@ jobs: - 'torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1' - 'torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0' - 'torch==2.9.1 torchvision==0.24.1 torchaudio==2.9.1' - include: - - py-version: '3.14' - pytorch-version: 'torch==2.9.1 torchvision==0.24.1 torchaudio==2.9.1' - install-options: '.' + # include: + # - py-version: '3.14' + # pytorch-version: 'torch==2.9.1 torchvision==0.24.1 torchaudio==2.9.1' + # install-options: '.' exclude: - py-version: '3.13' pytorch-version: 'numpy==1.26 torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2' diff --git a/.github/workflows/inactivity.yml b/.github/workflows/inactivity.yml index 1f42303c09..6baa0aa47b 100644 --- a/.github/workflows/inactivity.yml +++ b/.github/workflows/inactivity.yml @@ -31,3 +31,5 @@ jobs: stale-pr-message: "This pull request is stale because it has been open for 60 days with no activity." close-pr-message: "This pull request was closed because it has been inactive for 60 days since being marked as stale." repo-token: ${{ secrets.GITHUB_TOKEN }} + exempt-issue-labels: "epic,discussion,good first issue,RFC,student project" + exempt-pr-labels: "epic,discussion,good first issue,RFC,student project" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83ed5cba75..05823b5753 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,11 +45,3 @@ repos: # both pre-commit and pre-push supported # - id: talisman-push - id: talisman-commit - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.11.0.1 - hooks: - - id: shellcheck - #- repo: https://github.com/jumanjihouse/pre-commit-hooks - # rev: 3.0.0 - # hooks: - # - id: shellcheck diff --git a/.talismanrc b/.talismanrc new file mode 100644 index 0000000000..976c575f94 --- /dev/null +++ b/.talismanrc @@ -0,0 +1,3 @@ +threshold: medium +allowed_patterns: +- 'uses: [A-Za-z-\/]+@[\w\d]+' diff --git a/CHANGELOG.md b/CHANGELOG.md index 818a7dde09..a4bf58db96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,44 @@ +# v1.7.0 + +## Highlights + +1) DistributedSampler for efficient data loading and shuffling across multiple nodes with PyTorch +2) Randomized Symmetric eignevalue decomposition (eigh) +3) Incremental SVD directly from an HDF5 file +4) Partial support of the Array API Standard (version: '2020.10'), and API namespace under `x.__array_namespace__(api_version='2020.10')` +5) Distributed PTP (peak to peak) function + +*SVD, PCA, and DMD have been implemented within the project ESAPCA funded by the European Space Agency (ESA). This support is gratefully acknowledged.* + +## Changes +### Features +* Randomized Symmetric Eigenvalue Decomposition (eigh) by @mrfh92 in https://github.com/helmholtz-analytics/heat/pull/1964 +* Incremental SVD directly from HDF5 file by @LScheib in https://github.com/helmholtz-analytics/heat/pull/2005 +* Distributed Peak to Peak (ptp) function by @ivansherbakov9 in https://github.com/helmholtz-analytics/heat/pull/1954 +* PyTorch compatible DistributedSampler by @Berkant03 in https://githubcom/helmholtz-analytics/heat/pull/1807 + +### Interoperability +* Support Pytorch 2.9.1 by @github-actions[bot] in https://github.com/helmholtz-analytics/heat/pull/2001 +* Array API Namespace by @mtar in https://github.com/helmholtz-analytics/heat/pull/1022 + +### Bug Fixes + +* Sturdier MPI+GPU compatibility check by @JuanPedroGHM in https://github.com/helmholtz-analytics/heat/pull/1979 +* Fix handling of zarr groups by @ClaudiaComito in https://github.com/helmholtz-analytics/heat/pull/1990 +* Supporting negative indices for flip operations by @Marc-Jindra in https://github.com/helmholtz-analytics/heat/pull/2014 +* Fixed issue where matrices returned by ```eigh``` were not on the expected device by @GioPede in https://github.com/helmholtz-analytics/heat/pull/2046 +* Fixed issue where matrices returned by ```qr``` were not on the expected device by @GioPede in https://github.com/helmholtz-analytics/heat/pull/2045 +* Dtype is now set correctly when loading and saving hdf5 files by @Marc-Jindra in https://github.com/helmholtz-analytics/heat/pull/2014 +* Fix MPI large count issues when respliting by @JuanPedroGHM in https://github.com/helmholtz-analytics/heat/pull/1973 +* Default GPU+MPI compatibility settings for unknown MPI implementations by @JuanPedroGHM in https://github.com/helmholtz-analytics/heat/pull/2060 + + +## Contributors +@Marc-Jindra, @ClaudiaComito, @JuanPedroGHM, @GioPede, @ivansherbakov9, @LScheib, @Berkant03, @mrfh92, @mtar + +#### Acknowledgement and Disclaimer +*This work is partially carried out under a [programme](https://activities.esa.int/index.php/4000144045) of, and funded by, the European Space Agency. Any view expressed in this repository or related publications can in no way be taken to reflect the official opinion of the European Space Agency.* + # v1.6.0 ## Highlights diff --git a/CITATION.cff b/CITATION.cff index b09e7f80a5..225160bac8 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -7,19 +7,21 @@ message: 'If you use this software, please cite it as below.' type: software authors: # release highlights - - family-names: Hoppe - given-names: Fabian - - family-names: Gutiérrez Hermosillo Muriedas - given-names: Juan Pedro - family-names: Palazoglu given-names: Berkant - - family-names: Fischer - given-names: Carola + - family-names: Hoppe + given-names: Fabian + - family-names: Scheib + given-names: Lukas + - family-names: Tarnawa + given-names: Michael +# active contributors in alphabetic order - family-names: Akdag given-names: Hakan - family-names: Comito given-names: Claudia -# active contributors in alphabetic order + - family-names: Gutiérrez Hermosillo Muriedas + given-names: Juan Pedro - family-names: Hees given-names: Jörn - family-names: Jindra @@ -30,10 +32,6 @@ authors: given-names: Kai - family-names: Lemmen given-names: Jonas - - family-names: Scheib - given-names: Lukas - - family-names: Tarnawa - given-names: Michael # historic core team - family-names: Coquelin given-names: Daniel diff --git a/README.md b/README.md index 0f6ca711d1..9a0866ca69 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Heat is a distributed tensor framework for high performance data analytics. [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/helmholtz-analytics/heat/badge)](https://securityscorecards.dev/viewer/?uri=github.com/helmholtz-analytics/heat) [![OpenSSF Best Practices](https://bestpractices.coreinfrastructure.org/projects/7688/badge)](https://bestpractices.coreinfrastructure.org/projects/7688) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2531472.svg)](https://doi.org/10.5281/zenodo.2531472) -[![Benchmarks](https://img.shields.io/badge/Grafana-Benchmarks-2ea44f)](https://57bc8d92-72f2-4869-accd-435ec06365cb.ka.bw-cloud-instance.org:3000/d/adjpqduq9r7k0a/heat-cb?orgId=1) +[![Benchmarks](https://img.shields.io/badge/Grafana-Benchmarks-2ea44f)](https://930000e0-e69a-4939-912e-89a92316b420.ka.bw-cloud-instance.org/grafana) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![JuRSE Code Pick of the Month](https://img.shields.io/badge/JuRSE_Code_Pick-August_2024-blue)](https://www.fz-juelich.de/en/rse/jurse-community/jurse-code-of-the-month/august-2024) @@ -118,6 +118,7 @@ computational and memory needs of your laptop and desktop. ### Parallel I/O - h5py - netCDF4 +- zarr ### GPU support In order to do computations on your GPU(s): @@ -132,10 +133,10 @@ On most HPC-systems you will not be able to install/compile MPI or CUDA/ROCm you Install the latest version with ```bash -pip install heat[hdf5,netcdf] +pip install heat[hdf5,netcdf,zarr] ``` where the part in brackets is a list of optional dependencies. You can omit -it, if you do not need HDF5 or NetCDF support. +it, if you do not need HDF5, NetCDF, or Zarr support. ## **conda** diff --git a/doc/source/tutorials/notebooks/0_setup/0_setup_haicore.ipynb b/doc/source/tutorials/notebooks/0_setup/0_setup_haicore.ipynb index 6e4662a701..90758679bd 100644 --- a/doc/source/tutorials/notebooks/0_setup/0_setup_haicore.ipynb +++ b/doc/source/tutorials/notebooks/0_setup/0_setup_haicore.ipynb @@ -36,11 +36,7 @@ "tags": [] }, "source": [ - "\n", - "\n", - "\n", "## Introduction\n", - "---\n", "
\n", "Note:\n", "This notebook expects that you will be working on the JupyterLab hosted in HAICORE, at the Karlsruhe Institute of Technology.\n", diff --git a/doc/source/tutorials/notebooks/0_setup/0_setup_jsc.ipynb b/doc/source/tutorials/notebooks/0_setup/0_setup_jsc.ipynb index ee00ae6115..9ad18751a9 100644 --- a/doc/source/tutorials/notebooks/0_setup/0_setup_jsc.ipynb +++ b/doc/source/tutorials/notebooks/0_setup/0_setup_jsc.ipynb @@ -20,13 +20,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n", "
\n", " \n", "
\n", "\n", "## Introduction\n", - "---\n", "
\n", "Note:\n", "This tutorial is designed to run on Jupyter-JSC, a JupyterLab environment provided by the Jülich Supercomputing Centre. \n", @@ -156,7 +154,6 @@ "metadata": {}, "source": [ "## What is Heat for?\n", - "---\n", "\n", "[**deRSE24 NOTE**: do attend Fabian Hoppe's talk [TODAY at 16:30](https://events.hifis.net/event/994/contributions/7940/) for more details, benchmarks, and an overview of the parallel Python ecosystem.] \n", "\n", diff --git a/docker/Dockerfile.release b/docker/Dockerfile.release index 8ead42996a..3e0849705b 100644 --- a/docker/Dockerfile.release +++ b/docker/Dockerfile.release @@ -1,5 +1,5 @@ ARG HEAT_VERSION=latest -ARG PYTORCH_IMG=25.07-py3 +ARG PYTORCH_IMG=25.08-py3 FROM nvcr.io/nvidia/pytorch:${PYTORCH_IMG} AS base COPY ./tzdata.seed /tmp/tzdata.seed diff --git a/docker/Dockerfile.source b/docker/Dockerfile.source index 93a017b359..84e102712a 100644 --- a/docker/Dockerfile.source +++ b/docker/Dockerfile.source @@ -1,4 +1,4 @@ -ARG PYTORCH_IMG=25.07-py3 +ARG PYTORCH_IMG=25.08-py3 ARG HEAT_BRANCH=main FROM nvcr.io/nvidia/pytorch:${PYTORCH_IMG} AS base diff --git a/heat/core/communication.py b/heat/core/communication.py index fb875bb46c..5ee875b5ad 100644 --- a/heat/core/communication.py +++ b/heat/core/communication.py @@ -13,8 +13,6 @@ from typing import Any, Callable, Optional, List, Tuple, Union -import torch.version - from .stride_tricks import sanitize_axis from ._config import GPU_AWARE_MPI diff --git a/heat/core/linalg/tests/test_basics.py b/heat/core/linalg/tests/test_basics.py index 6e6ecf5374..87120adcc8 100644 --- a/heat/core/linalg/tests/test_basics.py +++ b/heat/core/linalg/tests/test_basics.py @@ -363,7 +363,6 @@ def test_inv(self): self.assertEqual(ainv.device, a.device) self.assertTupleEqual(ainv.shape, a.shape) self.assertTrue(ht.allclose(ainv, ares, atol=atol)) - a = ht.random.random((20, 20), dtype=dtype, split=1) ainv = ht.linalg.inv(a) i = ht.eye(a.shape, split=1, dtype=a.dtype) diff --git a/heat/core/tests/test_io.py b/heat/core/tests/test_io.py index 0ec1bd044a..45a3776758 100644 --- a/heat/core/tests/test_io.py +++ b/heat/core/tests/test_io.py @@ -4,7 +4,6 @@ import os import torch import tempfile -import time import random import shutil import fnmatch diff --git a/heat/core/version.py b/heat/core/version.py index b81bec1221..ea6b038234 100644 --- a/heat/core/version.py +++ b/heat/core/version.py @@ -2,7 +2,7 @@ major: int = 1 """Indicates Heat's main version.""" -minor: int = 7 +minor: int = 8 """Indicates feature extension.""" micro: int = 0 """Indicates revisions for bugfixes.""" diff --git a/scripts/numpy_coverage_tables.py b/scripts/numpy_coverage_tables.py index 59b80f1b2f..f431381288 100644 --- a/scripts/numpy_coverage_tables.py +++ b/scripts/numpy_coverage_tables.py @@ -486,3 +486,35 @@ # Print the Markdown table f.write(markdown_table) f.write("\n") + + +# for i, function_list in enumerate(numpy_functions): +# f.write(f"## {headers[str(i)]}\n") +# # Initialize a list to store the rows of the Markdown table +# table_rows = [] + +# # Check if functions exist in the heat library and create table rows +# for func_name in function_list: +# if ( +# hasattr(heat, func_name) +# or hasattr(heat.linalg, func_name.replace("linalg.", "")) +# or hasattr(heat.random, func_name.replace("random.", "")) +# ): +# support_status = "✅" # Green checkmark for supported functions +# else: +# support_status = "❌" # Red cross for unsupported functions + +# table_row = f"| {func_name} | {support_status} |" +# table_rows.append(table_row) + +# # Create the Markdown table header +# table_header = f"| {headers[str(i)]} | Heat |\n|---|---|\n" + +# # Combine the header and table rows +# markdown_table = table_header + "\n".join(table_rows) + +# # write link to table of contents +# f.write("[Back to Table of Contents](#table-of-contents)\n\n") +# # Print the Markdown table +# f.write(markdown_table) +# f.write("\n")