feat: update application directory logic (#15)

* use persistent application directory by default

* formatting wheels.yaml

* update readme with data dir info

* add retry to downloading datasets on ci

Author: lobis

Diff for: .github/workflows/build-test.yaml

@@ -19,13 +19,13 @@ on:
       [main]
 
       ## Paste this snippet into the workflow file to enable tmate debugging
-      #      - name: Setup tmate session
-      #        uses: mxschmitt/action-tmate@v3
-      #        if:
-      #          ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled
-      #          }}
-      #        with:
-      #          limit-access-to-actor: false
+    #      - name: Setup tmate session
+    #        uses: mxschmitt/action-tmate@v3
+    #        if:
+    #          ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled
+    #          }}
+    #        with:
+    #          limit-access-to-actor: false
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -185,7 +185,7 @@ jobs:
       - name: Move Geant4 datasets (Ubuntu/MacOS)
         if: matrix.python-version != '3.11' && matrix.os != 'windows-latest'
         run: |
-          GEANT4_DATA_DIR=$(python -c "import geant4_python_application; print(geant4_python_application.data_dir)")
+          GEANT4_DATA_DIR=$(python -c "import geant4_python_application; print(geant4_python_application.data_directory())")
           mkdir -p ${{ github.workspace }}/geant4-data $GEANT4_DATA_DIR
           mv ${{ github.workspace }}/geant4-data/* $GEANT4_DATA_DIR
 
@@ -194,14 +194,19 @@ jobs:
         if: matrix.python-version != '3.11' && matrix.os == 'windows-latest'
         shell: powershell
         run: |
-          $env:GEANT4_DATA_DIR = python -c "import geant4_python_application; print(geant4_python_application.data_dir)"
+          $env:GEANT4_DATA_DIR = python -c "import geant4_python_application; print(geant4_python_application.data_directory())"
           New-Item -ItemType Directory -Path "${{ github.workspace }}\geant4-data" -Force
           New-Item -ItemType Directory -Path $env:GEANT4_DATA_DIR -Force
           Move-Item "${{ github.workspace }}\geant4-data\*" $env:GEANT4_DATA_DIR -Force
 
-      - name: Check import
-        run: |
-          python -c "import geant4_python_application; geant4_python_application.Application()"
+      - name: Install datasets
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 30
+          retry_wait_seconds: 60
+          max_attempts: 5
+          command: |
+            python -c "import geant4_python_application as g4; g4.install_datasets()"
 
       - name: Run tests
         run: |

Diff for: .github/workflows/wheels.yaml

@@ -1,16 +1,15 @@
 name: Wheels
 
 on:
+  schedule:
+    - cron: "0 4 * * *" # every day at 4am UTC
   workflow_dispatch:
     inputs:
       publish_to_test_pypi:
         type: boolean
         description: Publish to Test PyPI
         default: false
 
-  schedule:
-    - cron: "0 4 * * *"
-
 env:
   GEANT4_VERSION: 11.2.0
   XERCES_VERSION: 3.2.4

Diff for: README.md

@@ -37,22 +37,63 @@ directory automatically during simulation startup.
 pip install -i https://test.pypi.org/simple/ geant4-python-application==0.0.2.dev1
 ```
 
-## Usage
+### Geant4 data files
+
+The Python wheels for this package do not contain any Geant4 data files due to
+their size. These files will be downloaded on demand without the need of any
+user interaction.
+
+The default location for these files can be obtained by calling:
+
+```bash
+python -c "import geant4_python_application; print(geant4_python_application.get_data_path())"
+```
+
+**Uninstalling the package will not remove the data files. The user is
+responsible for removing them manually.**
+
+#### Overriding the default data path
+
+The default data path can be overridden by calling `application_directory`. This
+should be done before the application is initialized.
+
+```python
+from geant4_python_application import application_directory
+
+application_directory("/some/other/path")
+```
+
+Overriding the default data directory is encouraged when submitting batch jobs
+to a cluster in order to avoid downloading the data files multiple times. In
+this case the application directory should point to some shared location.
+
+#### Using a temporary directory
+
+A temporary directory can be used by calling:
 
 ```python
-from geant4_python_application import Application, basic_gdml
+from geant4_python_application import application_directory
 
-# The data files will be downloaded to a temporary directory the first time this is called
-app = Application()
+application_directory(temp=True)
+```
 
-app.setup_manager(n_threads=4)
-app.setup_physics()
-app.setup_detector(gdml=basic_gdml)
-app.setup_action()
+The operating system will take care of cleaning up the temporary directory.
+
+It is possible that the operating system will delete only some of the files in
+the temporary directory which can lead to a Geant4 runtime error. In this case
+it's recommended to delete the temporary directory manually and let the
+application recreate it again.
+
+## Usage
+
+```python
+import geant4_python_application as g4
 
-app.initialize()
+# Use a temporary directory for the Geant4 data files (remove this line to use the default location)
+g4.application_directory(temp=True)
 
-events = app.run(n_events=100)
+with g4.Application(gdml=g4.basic_gdml, seed=137) as app:
+    events = app.run(n_events=100)
 
 print(events)
 ```

Diff for: pyproject.toml

@@ -37,6 +37,7 @@ dependencies = [
     "awkward-pandas",
     "numpy",
     "requests",
+    "platformdirs",
     "tqdm",
 ]
 description = "Geant4 Python Application"

Diff for: src/geant4_python_application/__init__.py

@@ -8,8 +8,9 @@
     pybind11_version,
 )
 from geant4_python_application.application import Application
-from geant4_python_application.datasets import data_dir, install_datasets
 from geant4_python_application.detector import Detector
+from geant4_python_application.files.datasets import data_directory, install_datasets
+from geant4_python_application.files.directories import application_directory
 from geant4_python_application.gdml import basic_gdml
 
 version = __version__
@@ -24,6 +25,7 @@
     "Application",
     "Detector",
     "basic_gdml",
-    "data_dir",
     "install_datasets",
+    "data_directory",
+    "application_directory",
 ]

Diff for: src/geant4_python_application/application.py

@@ -7,7 +7,6 @@
 import awkward as ak
 
 import geant4_python_application
-import geant4_python_application.datasets
 import geant4_python_application.events
 from geant4_python_application._geant4_application import (
     Application as Geant4Application,
@@ -71,7 +70,7 @@ class Application:
     def __init__(
         self, n_threads: int = 0, gdml: str = None, physics=None, seed: int = 0
     ):
-        geant4_python_application.datasets.install_datasets(show_progress=True)
+        geant4_python_application.install_datasets(show_progress=True)
 
         self._pipe, child_pipe = multiprocessing.Pipe()
         self._process = multiprocessing.Process(

Diff for: src/geant4_python_application/files/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

Diff for: src/geant4_python_application/datasets.py renamed to src/geant4_python_application/files/datasets.py

@@ -16,17 +16,21 @@
 
 url = "https://cern.ch/geant4-data/datasets"
 
+
 # It is discouraged to use the package directory to store data
 # data_dir = os.path.join(os.path.dirname(__file__), "geant4/data")
 # another idea is to use 'platformdirs' to store data in a platform-specific location
 
-data_dir = os.path.join(
-    tempfile.gettempdir(),
-    geant4_python_application.__name__,
-    "geant4",
-    geant4_python_application.geant4_version,
-    "data",
-)
+
+def data_directory() -> str:
+    return os.path.join(
+        geant4_python_application.application_directory(),
+        geant4_python_application.__name__,
+        "geant4",
+        geant4_python_application.geant4_version,
+        "data",
+    )
+
 
 # the datasets versions should be updated with each Geant4 version
 # https://geant4.web.cern.ch/download/11.2.0.html#datasets
@@ -155,27 +159,28 @@ def _download_extract_dataset(dataset: Dataset, pbar: tqdm):
 
         f.seek(0)
         with tarfile.open(fileobj=f, mode="r:gz") as tar:
-            tar.extractall(data_dir)
+            tar.extractall(data_directory())
 
 
 def install_datasets(force: bool = False, show_progress: bool = True):
-    os.environ["GEANT4_DATA_DIR"] = data_dir
+    os.environ["GEANT4_DATA_DIR"] = data_directory()
     datasets_to_download = []
     for dataset in datasets:
-        path = os.path.join(data_dir, dataset.name + dataset.version)
+        path = os.path.join(data_directory(), dataset.name + dataset.version)
         os.environ[dataset.env] = path
         if not os.path.exists(path) or force:
             datasets_to_download.append(dataset)
 
     if len(datasets_to_download) == 0:
         return
 
-    os.makedirs(data_dir, exist_ok=True)
+    os.makedirs(data_directory(), exist_ok=True)
     if show_progress:
         print(
             f"""
-Geant4 datasets (<2GB) will be installed to temporary directory {data_dir}
+Geant4 datasets (<2GB) will be installed to {data_directory()}
 This may take a while but only needs to be done once.
+You can override the default location by calling `application_directory(path)` or `application_directory(temp=True)` to use a temporary directory.
 The following Geant4 datasets will be installed: {", ".join([f"{dataset.name}@v{dataset.version}" for dataset in datasets_to_download])}"""
         )
 
@@ -196,14 +201,14 @@ def install_datasets(force: bool = False, show_progress: bool = True):
             concurrent.futures.wait(futures)
 
     if show_progress:
-        total_size_gb = sum(fp.stat().st_size for fp in Path(data_dir).rglob("*")) / (
-            1024**3
-        )
+        total_size_gb = sum(
+            fp.stat().st_size for fp in Path(data_directory()).rglob("*")
+        ) / (1024**3)
         print(f"Geant4 datasets size on disk after extraction: {total_size_gb:.2f}GB")
 
 
 def uninstall_datasets():
-    dir_to_remove = os.path.dirname(data_dir)
+    dir_to_remove = os.path.dirname(data_directory())
     package_dir = os.path.dirname(__file__)
 
     if not os.path.relpath(package_dir, dir_to_remove).startswith(".."):

Diff for: src/geant4_python_application/files/directories.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import os
+import pathlib
+import tempfile
+
+import platformdirs
+
+import geant4_python_application
+
+_app_name = geant4_python_application.__name__
+# TODO: get from pyproject.toml
+_app_author = "lobis"
+
+_dirs = platformdirs.AppDirs(_app_name, _app_author)
+
+_application_directory = _dirs.user_data_dir
+
+
+def application_directory(path: str | None = None, *, temp: bool = False) -> str:
+    global _application_directory
+
+    if temp and path:
+        raise ValueError("Cannot set both temp and path options")
+    if temp:
+        _application_directory = tempfile.gettempdir()
+        return _application_directory
+    if path:
+        # override the default application directory
+        # make sure path exists, otherwise create it. Throw if it cannot be created
+        pathlib.Path(path).mkdir(parents=True, exist_ok=True)
+        if not os.path.isdir(path):
+            raise ValueError(f"Cannot create application directory: {path}")
+        _application_directory = path
+
+    return _application_directory