railv2-bug-fixes

.github/workflows/run-create-interactive-structure.yml

@@ -0,0 +1,94 @@
+# This workflow will install Python dependencies, the supported rail packages, and run the create_interactive_structure.py script
+# If it fails with exit code = 2, that means that there have been updates to any of the stages 
+# this usually means that you just need to create a new branch, run create_interactive_structure.py and do a PR with rail_base to update the .pyi 
+# stub files 
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Running create_interactive_structure script
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-tags: true
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: System deps (MPI + build)
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends \
+        build-essential libbz2-dev \
+        openmpi-bin libopenmpi-dev
+    - name: Install dependencies
+      run: |
+        # sudo apt-get update
+        # python -m pip install --upgrade pip
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install "numpy<2" cython
+        python -m pip install --upgrade --no-binary=mpi4py mpi4py
+        python -m pip install --no-build-isolation somoclu
+        python -m pip install --upgrade pip
+        pip install wheel numpy
+        pip install .
+        pip install .[dev]
+        pip install .[algos]
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+    # runs the create interactive structure test to see if the interactive structure needs to be changed
+    # returns a 1 if something went wrong with the script, or a 2 if the files need to be changed
+    - name: Run create_interactive_structure.py 
+      run: | 
+        python create_interactive_structure.py
+    - name: Send status to Slack app (RAIL CI Reporter)
+      if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+      id: slack
+      uses: slackapi/slack-github-action@v1.24.0
+      with:
+        # For posting a rich message using Block Kit
+        payload: |
+          {
+            "blocks": [
+              {
+                "type": "header",
+                "text": {
+                  "type": "plain_text",
+                  "text": "${{ github.repository }} create_interactive_structure workflow"
+                }
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "GitHub Action build result: *${{ job.status }}* :${{ job.status }}:"
+                }
+              },
+              {
+                "type": "divider"
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                }
+              }
+            ]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

.github/workflows/smoke-test.yml

@@ -34,3 +34,42 @@ jobs:
     - name: Run unit tests with pytest
       run: |
         python -m pytest tests
+
+    - name: Send status to Slack app (RAIL CI Reporter)
+      if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+      id: slack
+      uses: slackapi/slack-github-action@v1.24.0
+      with:
+        # For posting a rich message using Block Kit
+        payload: |
+          {
+            "blocks": [
+              {
+                "type": "header",
+                "text": {
+                  "type": "plain_text",
+                  "text": "${{ github.repository }}"
+                }
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "GitHub Action build result: *${{ job.status }}* :${{ job.status }}:"
+                }
+              },
+              {
+                "type": "divider"
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                }
+              }
+            ]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

.github/workflows/testing-and-coverage.yml

@@ -4,6 +4,8 @@
 name: Unit test and code coverage
 
 on:
+  schedule:
+    - cron: 45 7 * * 1
   push:
     branches: [ main ]
   pull_request:
@@ -12,12 +14,15 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest 
     strategy:
       matrix:
         python-version: ['3.10', '3.11', '3.12']
     steps:
     - uses: actions/checkout@v3
+      with:
+        fetch-tags: true
+        fetch-depth: 0
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -26,6 +31,7 @@ jobs:
       run: |
         sudo apt-get update
         python -m pip install --upgrade pip
+        pip install wheel numpy
         pip install .
         pip install .[dev]
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

create_interactive_structure.py

@@ -153,9 +153,14 @@ def write_modules() -> None:
 
 def write_stubs() -> None:
     for module in interactive_modules:
-        module = "rail.interactive." + module
-        importlib.import_module(module)
-        _initialize_interactive_module(module, write_stubs=True)
+        stub_directory = InteractiveModule(
+            subfolder=module.replace(".", "/")
+        ).path.parent
+        full_module = "rail.interactive." + module
+        importlib.import_module(full_module)
+        _initialize_interactive_module(
+            full_module, write_stubs=True, stub_directory=stub_directory
+        )
 
 
 def store_pyi(delete: bool) -> dict[Path, str | None]:

pyproject.toml

@@ -41,6 +41,25 @@ dev = [
     "isort",         # as above
 ]
 
+algos = [
+    "pz-rail-astro-tools",
+    "pz-rail-bpz",
+    "pz-rail-cmnn",
+    "pz-rail-dnf",
+    "pz-rail-dsps",
+    "pz-rail-flexzboost",
+    "pz-rail-fsps",
+    "pz-rail-gpz-v1",
+    "pz-rail-pzflow",
+    "pz-rail-sklearn",
+    "pz-rail-som",
+    "pz-rail-yaw",
+    "pz-rail-lephare",
+    "pz-rail-tpz",
+    "pz-rail-delight",
+    "pz-rail-pipelines",
+]
+
 [project.scripts]
 rail = "rail.cli.rail.commands:cli"
 
@@ -56,7 +75,7 @@ write_to = "src/rail/core/_version.py"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
-addopts   = ["--cov=rail", "--cov-report=html"]
+addopts = ["--cov=rail", "--cov-report=html"]
 
 [tool.pylint]
 disable = [
@@ -82,16 +101,16 @@ max-branches = 25
 max-public-methods = 50
 
 [tool.mypy]
-disallow_untyped_defs    = true
+disallow_untyped_defs = true
 disallow_incomplete_defs = true
-ignore_missing_imports   = true
-local_partial_types      = true
-no_implicit_reexport     = true
-show_error_codes         = true
-strict_equality          = true
-warn_redundant_casts     = true
-warn_unreachable         = true
-warn_unused_ignores      = true
+ignore_missing_imports = true
+local_partial_types = true
+no_implicit_reexport = true
+show_error_codes = true
+strict_equality = true
+warn_redundant_casts = true
+warn_unreachable = true
+warn_unused_ignores = true
 
 
 [tool.coverage.run]

src/rail/evaluation/metrics/cdeloss.py

@@ -29,7 +29,7 @@ def evaluate(self) -> stat_and_pval:
         """
 
         # Calculate first term E[\int f*(z | X)^2 dz]
-        term1 = np.mean(np.trapz(self._pdfs**2, x=self._xvals))
+        term1 = np.mean(np.trapezoid(self._pdfs**2, x=self._xvals))
         # z bin closest to ztrue
         nns = [np.argmin(np.abs(self._xvals - z)) for z in self._ztrue]
         # Calculate second term E[f*(Z | X)]

src/rail/evaluation/metrics/tomography.py

@@ -91,7 +91,7 @@ def run(self) -> None:
                     p_j = kde_j(eval_grid)
 
                     # Compute overlap as the integral of the minimum of both distributions
-                    overlap = np.trapz(np.minimum(p_i, p_j), eval_grid)
+                    overlap = np.trapezoid(np.minimum(p_i, p_j), eval_grid)
                     overlap_matrix[i, j] = overlap
                     overlap_matrix[j, i] = overlap  # Symmetric matrix
 

src/rail/interactive/creation/degraders/grid_selection.pyi

@@ -97,12 +97,10 @@ def grid_selection(**kwargs) -> Any:
         Default: 100.0
     ratio_file : str, optional
         path to ratio file
-        Default: rail_astro_tools/src/rail/examples_data/creation_data/data/hsc_ratios_a
-        nd_specz.hdf5
+        Default: rail/examples_data/creation_data/data/hsc_ratios_and_specz.hdf5
     settings_file : str, optional
         path to pickled parameters file
-        Default:
-        rail_astro_tools/src/rail/examples_data/creation_data/data/HSC_grid_settings.pkl
+        Default: rail/examples_data/creation_data/data/HSC_grid_settings.pkl
     random_seed : int, optional
         random seed for reproducibility
         Default: 12345

src/rail/interactive/creation/degraders/observing_condition_degrader.pyi

@@ -82,12 +82,12 @@ def obs_condition(**kwargs) -> Any:
         Default: 128
     mask : str, optional
         mask for the input maps in HEALPIX format.
-        Default: rail_astro_tools/src/rail/examples_data/creation_data/data/survey_condi
-        tions/DC2-mask-neg-nside-128.fits
+        Default: rail/examples_data/creation_data/data/survey_conditions/DC2-mask-neg-
+        nside-128.fits
     weight : str, optional
         weight for assigning pixels to galaxies in HEALPIX format.
-        Default: rail_astro_tools/src/rail/examples_data/creation_data/data/survey_condi
-        tions/DC2-dr6-galcounts-i20-i25.3-nside-128.fits
+        Default: rail/examples_data/creation_data/data/survey_conditions/DC2-dr6-
+        galcounts-i20-i25.3-nside-128.fits
     tot_nVis_flag : bool, optional
         flag indicating whether nVisYr is the total or average per year if supplied.
         Default: True