Merge branch 'master' into soap_docs

Author: robjmcgibbon

source/_static/viewer.js

@@ -344,35 +344,41 @@ async function display_directory(path, object) {
     // Get directory size as a string
     const dir_size = format_file_size(object.size);
 
-    if(sanitize_path(path) != "") {
+    if((sanitize_path(path) != "") && (object.size < 1125899906842624)) {
         // Add a link to download the directory
         const card = add_element(div, "div");
         card.className = "downloadoptions";
         const details = add_element(card, "details");
         const summary = add_element(details, "summary");
         add_text(summary, "Download options");
-        // Make a list of access options
-        const ul = add_element(details, "ul");
-        // Full download
-        const li1 = add_element(ul, "li");
-        const dl_link = add_element(li1, "a");
-        dl_link.href = download_url(path);
-        add_text(dl_link, "Full directory download as .tar file");
         if(object.size >= 107374182400) {
             if(Object.keys(object.directories).length > 0) {
-                add_text(li1, " ( ⚠️"+dir_size+", see subdirectories for smaller downloads)");
+                add_text(summary, " ( ⚠️"+dir_size+", see subdirectories for smaller downloads)");
             } else {
-                add_text(li1, " ( ⚠️"+dir_size+")");
+                add_text(summary, " ( ⚠️"+dir_size+")");
             }
         } else {
-            add_text(li1, " ("+dir_size+")");
+            add_text(summary, " ("+dir_size+")");
         }
+        // Make a list of access options
+        const ul = add_element(details, "ul");
         // Access via api
-        const li2 = add_element(ul, "li");
-        set_inner_html(li2, "Access individual datasets using the <a href='/flamingo/service_docs/python_module.html'>hdfstream python module</a>");
+        const li_api = add_element(ul, "li");
+        set_inner_html(li_api, "Access individual datasets using the <a href='/flamingo/service_docs/python_module.html'>hdfstream python module</a>");
+        // Full download
+        const li_dl = add_element(ul, "li");
+        const dl_link = add_element(li_dl, "a");
+        dl_link.href = download_url(path);
+        add_text(dl_link, "Full directory download as .tar file");
+        // Command line download
+        const li_cmd = add_element(ul, "li");
+        add_text(li_cmd, "Download and unpack on the command line:");
+        add_element(li_cmd, "br");
+        const dl_code = add_element(add_element(li_cmd, "pre"), "code");
+        dl_code.textContent = 'curl -u my_username -L "'+download_url(path)+'" | tar xvf -';
         // Access via globus
-        const li3 = add_element(ul, "li");
-        set_inner_html(li3, "Users with a <a href='https://cosma.readthedocs.io/en/latest'>Cosma</a> account can use <a href='https://cosma.readthedocs.io/en/latest/data.html#globus-online'>Globus Online or bbcp</a>");
+        const li_globus = add_element(ul, "li");
+        set_inner_html(li_globus, "Users with a <a href='https://cosma.readthedocs.io/en/latest'>Cosma</a> account can use <a href='https://cosma.readthedocs.io/en/latest/data.html#globus-online'>Globus Online or bbcp</a>");
     }
 
     // Make a div to contain any description for this diretcory

source/index.rst

@@ -7,6 +7,7 @@ Welcome to the FLAMINGO data release documentation
    introduction
    service_docs/index
    simulations/index
+   power_spectra
    snapshots/index
    Halo catalogues <soap/index>
    lightcones/index

source/lightcones/healpix_lightcone_io.rst

@@ -0,0 +1,87 @@
+Reading the maps with lightcone_io
+==================================
+
+The `lightcone_io <https://lightconeio.readthedocs.io/en/stable/>`__
+python module can be used to read FLAMINGO HEALPix maps, either by
+reading downloaded HDF5 files directly or by accessing maps stored on
+`Cosma <https://cosma.readthedocs.io/en/latest/>`__ using the
+`hdfstream <https://hdfstream-python.readthedocs.io/en/latest>`__
+service. The latter method might be better if you're only interested
+in certain quantities or regions on the sky.
+
+Installation
+------------
+
+The ``lightcone_io`` module can be installed as follows::
+
+  pip install lightcone_io
+
+For remote access to snapshots we also need the hdfstream module::
+
+  pip install hdfstream
+
+Opening a set of HEALPix maps
+-----------------------------
+
+The examples below show how to access the down-sampled maps for
+observer 0 in the fiducial ``L1_m9`` simulation.
+
+.. tab-set::
+
+   .. tab-item:: Opening remote files
+
+      .. code-block:: python
+
+         # Connect to the hdfstream service and open the root directory
+         import hdfstream
+         root = hdfstream.open("cosma", "/", user="my_username") # TODO: update when we remove access restrictions
+
+         # Location of the lightcone output relative to the directory we opened
+         basedir="FLAMINGO/L1_m9/L1_m9/healpix_maps/nside_4096"
+
+         # Specify which observer's lightcone to read
+         basename="lightcone0"
+
+         # Open the set of HEALPix maps on the server
+         import lightcone_io as lc
+         shell = lc.ShellArray(basedir, basename, remote_dir=root)
+
+   .. tab-item:: Opening local files
+
+      .. code-block:: python
+
+         # Location of the lightcone output we downloaded
+         basedir="./FLAMINGO/L1_m9/L1_m9/healpix_maps/nside_4096/"
+
+         # Specify which observer's lightcone to read
+         basename="lightcone0"
+
+         # Open the set of HEALPix maps
+         import lightcone_io as lc
+         shell = lc.ShellArray(basedir, basename)
+
+Reading map data
+----------------
+
+For each observer we have a number of :doc:`concentric shells
+<healpix_shell_redshifts>`, and for each shell we have maps of various
+:doc:`physical quantities <healpix_map_descriptions>`. The pixel data
+for a particular map can be accessed by specifying a shell index and
+the name of the map::
+
+  # Index of the shell we're interested in
+  shell_nr = 0
+
+  # Name of the quantity we want to read
+  map_name = "TotalMass"
+
+  # Read the data
+  map_data = shell[shell_nr][map_name][...]
+
+Here, indexing an individual map with the ellipsis (``[...]``)
+triggers the full map to be read in or downloaded. The result is a
+`unyt <https://unyt.readthedocs.io/en/stable/>`__ array containing the
+pixel values with unit information. See the `lightcone_io
+documentation
+<https://lightconeio.readthedocs.io/en/stable/healpix_maps.html>`__
+for more details.

source/lightcones/healpix_lightcones.rst

@@ -50,6 +50,7 @@ information see the documentation links below and appendix A of
    File format <healpix_format>
    Shell redshifts <healpix_shell_redshifts>
    Map descriptions <healpix_map_descriptions>
+   Reading the maps <healpix_lightcone_io>
    integrated_lightcones
 
 
@@ -65,4 +66,3 @@ information see the documentation links below and appendix A of
 
 
 
-

source/lightcones/integrated_lightcones.rst

@@ -65,17 +65,35 @@ the gravitational potential. The files correspond to integrated weak
 lensing convergence maps (:math:`\kappa`) that assume an
 non-tomographic Euclid-like source redshift distribution and are saved
 as ring-ordered Healpix maps at :math:`N_\mathrm{side} = 8192`. No
-smoothing or noise has been applied to these files. The files can be
-read as
+smoothing or noise has been applied to these files.
 
+Each simulation has a subdirectory ``value_add/broxterman24`` which
+contains the convergence maps. For example, the maps for ``L1_m9`` are
+in `this directory
+</flamingo/viewer.html?path=FLAMINGO/L1_m9/L1_m9/value_add/broxterman24>`__. The
+files can be accessed using the :doc:`hdfstream
+</service_docs/python_module>` module or downloaded and read directly,
+as shown below.
 
-.. code-block:: python
+.. tab-set::
 
-    import h5py
-    file_path = '/cosma8/data/dp004/dc-brox1/FLAMINGO_datarelease/WL_convergence_Euclid_like_nz_Broxterman24_L1_m9_lc0.hdf5'
-    data_file = h5py.File(f"/{file_path}",'r')
-    kappa_map = data_file["Convergence"][:]
-    data_file.close()
+   .. tab-item:: Remote access
+
+      .. code-block:: python
+
+         import hdfstream
+         file_path = 'FLAMINGO/L1_m9/L1_m9/value_add/broxterman24/WL_convergence_Euclid_like_nz_Broxterman24_L1_m9_lc0.hdf5'
+         with hdfstream.open("cosma", file_path) as data_file:
+           kappa_map = data_file["Convergence"][:]
+
+   .. tab-item:: Reading downloaded files
+
+      .. code-block:: python
+
+         import h5py
+         file_path = './FLAMINGO/L1_m9/L1_m9/value_add/broxterman24/WL_convergence_Euclid_like_nz_Broxterman24_L1_m9_lc0.hdf5'
+         with h5py.File(f"/{file_path}",'r') as data_file:
+           kappa_map = data_file["Convergence"][:]
 
 where the part ``L1_m9_lc0`` changes between the variations and lightcones (lc), for example,
 ``L2p8_m9_DMO_lc4`` for observer four in the 2800 cGpc dark-matter-only box.
@@ -98,30 +116,63 @@ lightcone (of the 0th observer of each ``L1_m9`` simulation
 given). When integrating along the line of sight the on-sky
 coordinates of the gas particles in each shell are rotated as
 described in `Broxterman et al (2024)
-<https://ui.adsabs.harvard.edu/abs/2024MNRAS.529.2309B%2F/abstract>`__. The
-maps containing the X-ray emission from hot gas can be read as
+<https://ui.adsabs.harvard.edu/abs/2024MNRAS.529.2309B%2F/abstract>`__.
 
-.. code-block:: python
+Each simulation has a subdirectory ``value_add/mcdonald26`` which
+contains the convergence maps. For example, the maps for ``L1_m9`` are
+in `this directory
+</flamingo/viewer.html?path=FLAMINGO/L1_m9/L1_m9/value_add/mcdonald26>`__. The
+files can be accessed using the :doc:`hdfstream
+</service_docs/python_module>` module or downloaded and read
+directly. Below, we show how to read the maps:
 
-    import h5py
-    filename="/cosma8/data/dp004/dc-mcdo1/DataRelease/ROSAT_Xray_Maps/Gas_Convolved/{BoxsizeResolution}/{SimulationName}.h5"
-    xray_source="Gas"
-    map_name="XrayROSATIntrinsicPhotonsConvolved"
+.. tab-set::
+
+   .. tab-item:: Remote access
+
+      .. code-block:: python
 
-    with h5py.File(filename.format(BoxsizeResolution="L1000N1800", SimulationName="HYDRO_FIDUCIAL"), "r") as integrated_map:
+         import hdfstream
 
-        # read ROSAT convolved photon flux X-ray map
-        ROSAT_Xray_map=integrated_map[xray_source+'/'+map_name][:]
+         xray_source="Gas"
+         map_name="XrayROSATIntrinsicPhotonsConvolved"
+         with hdfstream.open("cosma", "./FLAMINGO/L1_m9/L1_m9/value_add/mcdonald26/ROSAT_convolved_Xray_AllSky_L1_m9.hdf5", "r") as integrated_map:
 
-        # print simulations identifier (name) in FLAMINGO papers:
-        print("\tFLAMINGO identifier: {label}".format(label=integrated_map[xray_source].attrs['paper_name'][:]))
+           # read ROSAT convolved photon flux X-ray map
+           ROSAT_Xray_map=integrated_map[xray_source+'/'+map_name][:]
 
-        # print integrated redshift range:
-        lc_zmin=integrated_map[xray_source].attrs['redshift_min'] lc_zmax=integrated_map[xray_source].attrs['redshift_max']
-        print("\tredshift range: {zmin:.3f}, {zmax:.3f}".format(zmin=lc_zmin, zmax=lc_zmax))
+           # print simulations identifier (name) in FLAMINGO papers:
+           print("\tFLAMINGO identifier: {label}".format(label=integrated_map[xray_source].attrs['paper_name'][:]))
 
-        # print expression for map units:
-        print("\tunit expression: {map_units}".format(map_units=integrated_map[xray_source].attrs['unit_expression']))
+           # print integrated redshift range:
+           lc_zmin=integrated_map[xray_source].attrs['redshift_min'] lc_zmax=integrated_map[xray_source].attrs['redshift_max']
+           print("\tredshift range: {zmin:.3f}, {zmax:.3f}".format(zmin=lc_zmin, zmax=lc_zmax))
+
+           # print expression for map units:
+           print("\tunit expression: {map_units}".format(map_units=integrated_map[xray_source].attrs['unit_expression']))
+
+   .. tab-item:: Reading local files
+
+      .. code-block:: python
+
+         import h5py
+
+         xray_source="Gas"
+         map_name="XrayROSATIntrinsicPhotonsConvolved"
+         with h5py.File("./FLAMINGO/L1_m9/L1_m9/value_add/mcdonald26/ROSAT_convolved_Xray_AllSky_L1_m9.hdf5", "r") as integrated_map:
+
+           # read ROSAT convolved photon flux X-ray map
+           ROSAT_Xray_map=integrated_map[xray_source+'/'+map_name][:]
+
+           # print simulations identifier (name) in FLAMINGO papers:
+           print("\tFLAMINGO identifier: {label}".format(label=integrated_map[xray_source].attrs['paper_name'][:]))
+
+           # print integrated redshift range:
+           lc_zmin=integrated_map[xray_source].attrs['redshift_min'] lc_zmax=integrated_map[xray_source].attrs['redshift_max']
+           print("\tredshift range: {zmin:.3f}, {zmax:.3f}".format(zmin=lc_zmin, zmax=lc_zmax))
+
+           # print expression for map units:
+           print("\tunit expression: {map_units}".format(map_units=integrated_map[xray_source].attrs['unit_expression']))
 
 
 
@@ -136,15 +187,13 @@ to per steradian (or square degree.)
     import h5py
     import healpy as hp
     import unyt
-    filename="/cosma8/data/dp004/dc-mcdo1/DataRelease/ROSAT_Xray_Maps/Gas_Convolved/{BoxsizeResolution}/{simulation}.h5"
     xray_source="Gas"
     map_name="XrayROSATIntrinsicPhotonsConvolved"
-    with h5py.File(filename.format(BoxsizeResolution="L1000N1800", simulation="HYDRO_FIDUCIAL"), "r") as integrated_map:
+
+    with h5py.File("./FLAMINGO/L1_m9/L1_m9/value_add/mcdonald26/ROSAT_convolved_Xray_AllSky_L1_m9.hdf5", "r") as integrated_map:
         # read map
         ROSAT_Xray_map_per_sr=integrated_map[xray_source+'/'+map_name][:]
-
         nside = integrated_map[xray_source].attrs['shell_nside'] # can take nside from map attributes, otherwise confirm from the number of pixels in the map
         # apply unit transformation and define map units
         ROSAT_Xray_map_per_sr /= hp.nside2pixarea(nside, degrees=False) * unyt.photon / unyt.s / unyt.radian**2
 
-

source/lightcones/particle_lightcones.rst

@@ -14,3 +14,4 @@ beyond which particles are not output.
    File format <particle_lightcone_format>
    Redshift ranges <particle_lightcone_redshifts>
    Particle properties <particle_lightcone_properties>
+   Reading the particles <reading_particles>