skyCatalogs_creator CI #90
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: skyCatalogs_creator CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - releases/* | |
| pull_request: | |
| branches: | |
| - main | |
| - releases/* | |
| schedule: | |
| - cron: 0 23 * * 4 | |
| jobs: | |
| rubin_sim_data: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest ] | |
| py: [ "3.13" ] | |
| CC: [ gcc ] | |
| CXX: [ g++ ] | |
| defaults: | |
| run: | |
| # cf. https://github.com/conda-incubator/setup-miniconda#important | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Cache files | |
| uses: actions/cache@v4 | |
| id: cache-rubin-sim-data | |
| env: | |
| cache-name: cache-rubin-sim-data | |
| with: | |
| path: rubin_sim_data/ | |
| key: rubin_sim_data_files | |
| - name: Install rubin_sim_data (if not cached) | |
| if: ${{ steps.cache-rubin-sim-data.outputs.cache-hit != 'true' }} | |
| run: | | |
| echo ${{ steps.cache-rubin-sim-data.outputs.cache-hit }} | |
| mkdir rubin_sim_data | |
| mkdir rubin_sim_data/sims_sed_library | |
| # Just get the throughputs and SED data for now. | |
| curl https://s3df.slac.stanford.edu/groups/rubin/static/sim-data/rubin_sim_data/throughputs_2023_09_07.tgz | tar -C rubin_sim_data -xz | |
| curl https://s3df.slac.stanford.edu/groups/rubin/static/sim-data/sed_library/seds_170124.tar.gz | tar -C rubin_sim_data/sims_sed_library -xz | |
| full-stack-build: | |
| needs: [rubin_sim_data] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest ] | |
| py: [ "3.13" ] | |
| CC: [ gcc ] | |
| CXX: [ g++ ] | |
| defaults: | |
| run: | |
| # cf. https://github.com/conda-incubator/setup-miniconda#important | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.py }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.py }} | |
| - name: Verify pytho and pip versions | |
| run: | | |
| python --version | |
| pip --version | |
| - name: Audit disk usage and remove unneeded tooling | |
| run: | | |
| df -h | |
| sudo rm -rf /usr/lib/jvm | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/.ghcup | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/local/chromium | |
| df -h | |
| - name: Setup conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| auto-update-conda: false | |
| conda-version: "25.11.1" | |
| activate-environment: stack | |
| python-version: ${{ matrix.py }} | |
| condarc-file: etc/.condarc | |
| - name: Load cached files | |
| uses: actions/cache@v4 | |
| id: cache-rubin-sim-data | |
| env: | |
| cache-name: cache-rubin-sim-data | |
| with: | |
| path: rubin_sim_data/ | |
| key: rubin_sim_data_files | |
| - name: Install rubin_sim_data (if not cached) | |
| if: ${{ steps.cache-rubin-sim-data.outputs.cache-hit != 'true' }} | |
| run: | | |
| mkdir rubin_sim_data | |
| mkdir rubin_sim_data/sims_sed_library | |
| # Just get the throughputs and SED data for now. | |
| curl https://s3df.slac.stanford.edu/groups/rubin/static/sim-data/rubin_sim_data/throughputs_2023_09_07.tgz | tar -C rubin_sim_data -xz | |
| curl https://s3df.slac.stanford.edu/groups/rubin/static/sim-data/sed_library/seds_170124.tar.gz | tar -C rubin_sim_data/sims_sed_library -xz | |
| - name: Install conda test deps | |
| run: | | |
| # conda update -n base conda | |
| conda info | |
| conda list | |
| conda install -y --file etc/conda_test_requirements.txt | |
| conda info | |
| - name: Install dependencies via conda | |
| run: | | |
| conda install --prefix /usr/share/miniconda/envs/stack -y --file etc/conda_requirements.txt | |
| - name: Install pystellibs | |
| run: | | |
| pip install --no-build-isolation --no-deps git+https://github.com/mfouesneau/pystellibs | |
| - name: Install astro-datalab | |
| run: | | |
| pip install --no-build-isolation --no-deps astro-datalab | |
| - name: Install skyCatalogs | |
| run: | | |
| pip install --no-build-isolation --no-deps git+https://github.com/LSSTDESC/skyCatalogs@v2.4.0 | |
| - name: Install skyCatalogs_creator | |
| run: | | |
| pip install --no-build-isolation --no-deps -e . | |
| - name: Run tests with the LSST Stack | |
| run: | | |
| export RUBIN_SIM_DATA_DIR=`pwd`/rubin_sim_data | |
| export SIMS_SED_LIBRARY_DIR=${RUBIN_SIM_DATA_DIR}/sims_sed_library | |
| export PYTHONPATH=`pwd`/pystellibs/src:`pwd`/skyCatalogs:${PYTHONPATH} | |
| echo $PYTHONPATH | |
| eups list lsst_distrib | |
| export SCRIPTS=skycatalogs_creator/scripts | |
| export CI_DATA=skycatalogs_creator/data/ci | |
| export CI_GCR=skycatalogs_creator/data/ci_sample | |
| export TRILEGAL_PIXEL=9556 | |
| python ${SCRIPTS}/create_main.py --help | |
| python ${SCRIPTS}/create_flux.py --help | |
| python ${SCRIPTS}/create_main.py trilegal --skycatalog-root \ | |
| ${CI_DATA} --pixels $TRILEGAL_PIXEL | |
| python ${SCRIPTS}/create_flux.py --object-type trilegal \ | |
| --skycatalog-root ${CI_DATA} --flux-parallel 24 --pixels $TRILEGAL_PIXEL | |
| pytest tests | |
| # Clean up after running compare tests | |
| rm ${CI_DATA}/*.parquet | |
| rm ${CI_DATA}/*.yaml |