notebooks/convert_h5dataframe2flatgeobuf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ada5ec98-fb08-4457-86be-307bfcfe551e",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "# Convert HDF5 to FlatGeobuf using H5DataFrame\n",
    "\n",
    "Loading data from a nested ICESat-2 ATL03 HDF5 file,\n",
    "and converting it to a [FlatGeobuf](https://flatgeobuf.org) format.\n",
    "\n",
    "Steps:\n",
    "1. Read a variable from HDF5 file into `H5DataFrame` class\n",
    "2. Create `geopandas.GeoDataFrame` with columns h_ph and geometry (longitude/latitude)\n",
    "3. Save GeoDataFrame to FlatGeobuf\n",
    "\n",
    "References:\n",
    "- https://github.com/MAAP-Project/gedi-subsetter/blob/0.6.0/src/gedi_subset/gedi_utils.py#L139-L381"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6e91818f-17fa-4ecf-be62-a3b1c7e4fae2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['USE_PYGEOS'] = '0'\n",
    "import geopandas as gpd\n",
    "import h5py\n",
    "import s3fs\n",
    "import tqdm\n",
    "\n",
    "try:\n",
    "    from gedi_subset.h5frame import H5DataFrame\n",
    "except ImportError:\n",
    "    !pip install git+https://github.com/MAAP-Project/gedi-subsetter.git@0.6.0\n",
    "    from gedi_subset.h5frame import H5DataFrame\n",
    "\n",
    "try:\n",
    "    import pyogrio\n",
    "except ImportError:\n",
    "    !mamba install -y pyogrio\n",
    "    import pyogrio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3958f3b8-eaee-4bd4-b354-6d67e2e09dc4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "SYSTEM INFO\n",
      "-----------\n",
      "python     : 3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]\n",
      "executable : /srv/conda/envs/notebook/bin/python\n",
      "machine    : Linux-5.10.167-147.601.amzn2.x86_64-x86_64-with-glibc2.27\n",
      "\n",
      "GEOS, GDAL, PROJ INFO\n",
      "---------------------\n",
      "GEOS       : 3.11.2\n",
      "GEOS lib   : None\n",
      "GDAL       : 3.7.0\n",
      "GDAL data dir: /srv/conda/envs/notebook/share/gdal\n",
      "PROJ       : 9.2.1\n",
      "PROJ data dir: /srv/conda/envs/notebook/share/proj\n",
      "\n",
      "PYTHON DEPENDENCIES\n",
      "-------------------\n",
      "geopandas  : 0.12.1\n",
      "numpy      : 1.23.5\n",
      "pandas     : 1.5.1\n",
      "pyproj     : 3.6.0\n",
      "shapely    : 2.0.1\n",
      "fiona      : 1.9.4\n",
      "geoalchemy2: None\n",
      "geopy      : None\n",
      "matplotlib : 3.6.2\n",
      "mapclassify: 2.5.0\n",
      "pygeos     : 0.14\n",
      "pyogrio    : 0.6.0\n",
      "psycopg2   : 2.9.6 (dt dec pq3 ext lo64)\n",
      "pyarrow    : 12.0.1\n",
      "rtree      : 1.0.1\n"
     ]
    }
   ],
   "source": [
    "gpd.show_versions()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ce9fd40-a352-479c-8a7d-607b86b34ab9",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "## List files to convert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "15dde64d-6586-47b0-b51c-a0372116453e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-08-08 23:45:34 7754735138 ATL03_20181120182818_08110112_006_02.h5\n",
      "2023-08-08 23:47:04 6997123664 ATL03_20190219140808_08110212_006_02.h5\n",
      "2023-08-08 23:47:04 6925710500 ATL03_20200217204710_08110612_006_01.h5\n",
      "2023-08-08 23:47:04 8392279594 ATL03_20211114142614_08111312_006_01.h5\n",
      "2023-08-08 23:47:04 7954039827 ATL03_20230211164520_08111812_006_01.h5\n"
     ]
    }
   ],
   "source": [
    "!aws s3 ls s3://nasa-cryo-scratch/h5cloud/original/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "30f0b71e-c877-4d71-8f1b-56ca3047f7e7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['nasa-cryo-scratch/h5cloud/original/ATL03_20181120182818_08110112_006_02.h5',\n",
       " 'nasa-cryo-scratch/h5cloud/original/ATL03_20190219140808_08110212_006_02.h5',\n",
       " 'nasa-cryo-scratch/h5cloud/original/ATL03_20200217204710_08110612_006_01.h5',\n",
       " 'nasa-cryo-scratch/h5cloud/original/ATL03_20211114142614_08111312_006_01.h5',\n",
       " 'nasa-cryo-scratch/h5cloud/original/ATL03_20230211164520_08111812_006_01.h5']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3 = s3fs.S3FileSystem(anon=False)\n",
    "s3urls = s3.glob(path=\"s3://nasa-cryo-scratch/h5cloud/original/*\")\n",
    "s3urls"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42f37fe5-1311-4a4e-9048-49eb83224828",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "## Single file conversion\n",
    "\n",
    "Showing how to process just one HDF5 file.\n",
    "Skip to bottom if running on multiple."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9db7d56-f03a-4342-99ea-c886a7cc14ff",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "### Read HDF5 variables into H5DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e78ad96f-d462-4032-89d6-8c1984b69644",
   "metadata": {},
   "outputs": [],
   "source": [
    "s3url = \"s3://nasa-cryo-scratch/h5cloud/original/ATL03_20230211164520_08111812_006_01.h5\"\n",
    "h5 = h5py.File(name=s3.open(path=s3url, mode=\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b6f42494-d847-4b56-a197-ec24132885ee",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<KeysViewHDF5 ['METADATA', 'ancillary_data', 'atlas_impulse_response', 'ds_surf_type', 'ds_xyz', 'gt1l', 'gt1r', 'gt2l', 'gt2r', 'gt3l', 'gt3r', 'orbit_info', 'quality_assessment']>\n",
      "delta_time\n",
      "dist_ph_across\n",
      "dist_ph_along\n",
      "h_ph\n",
      "lat_ph\n",
      "lon_ph\n",
      "pce_mframe_cnt\n",
      "ph_id_channel\n",
      "ph_id_count\n",
      "ph_id_pulse\n",
      "quality_ph\n",
      "signal_conf_ph\n",
      "weight_ph\n"
     ]
    }
   ],
   "source": [
    "# Print top-level groups\n",
    "print(h5.keys())\n",
    "\n",
    "# Print all nested groups (very slow)\n",
    "# h5.visit(func=lambda name: print(name))\n",
    "\n",
    "# Print specific groups (slow)\n",
    "h5[\"gt1l/heights\"].visit(func=lambda name: print(name))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74520fcb-90f8-47d5-8e0e-4304767074a6",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "#### Read in just the `gt1l/heights` group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8b42bc87-14d9-427a-8bbe-b0c5b36a0235",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty H5DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = H5DataFrame(group=h5[\"gt1l/heights\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c17069c-ab93-445c-9237-8d1752ab0606",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "It looks like there are no columns, because H5DataFrame is lazy,\n",
    "and you need to access the variable (e.g. `h_ph`) explicitly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c098a56f-80ff-403c-95f5-72428f04be1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0           267.050385\n",
       "1           279.333771\n",
       "2           272.782135\n",
       "3           226.964447\n",
       "4           228.495010\n",
       "               ...    \n",
       "84812149    286.131195\n",
       "84812150    282.127655\n",
       "84812151    278.603241\n",
       "84812152    155.884216\n",
       "84812153    151.234497\n",
       "Name: h_ph, Length: 84812154, dtype: float32"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"h_ph\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9acb10ec-fd95-4500-a15d-3ee52a3afd7b",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "### Create `geopandas.GeoDataFrame`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "152d93ea-33c4-443a-8e88-8ea64a03799a",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "#### Create geometry column with CRS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b1b0e1c8-bcdc-4bda-ac16-5965024a6751",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 46.2 s, sys: 11.4 s, total: 57.6 s\n",
      "Wall time: 1min 54s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "geometry = gpd.points_from_xy(x=df[\"lon_ph\"], y=df[\"lat_ph\"], crs=\"OGC:CRS84\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e535997-6c6b-4b27-9ea6-77f9ee8ea36b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<GeometryArray>\n",
       "[<POINT (-60.059 -79.006)>, <POINT (-60.059 -79.006)>,\n",
       " <POINT (-60.059 -79.006)>, <POINT (-60.059 -79.006)>,\n",
       " <POINT (-60.059 -79.006)>, <POINT (-60.059 -79.006)>,\n",
       " <POINT (-60.059 -79.006)>, <POINT (-60.059 -79.006)>,\n",
       " <POINT (-60.059 -79.006)>, <POINT (-60.059 -79.006)>,\n",
       " ...\n",
       " <POINT (-69.925 -50.011)>, <POINT (-69.925 -50.011)>,\n",
       " <POINT (-69.925 -50.011)>, <POINT (-69.925 -50.011)>,\n",
       " <POINT (-69.925 -50.011)>, <POINT (-69.925 -50.011)>,\n",
       " <POINT (-69.925 -50.011)>, <POINT (-69.925 -50.011)>,\n",
       " <POINT (-69.925 -50.011)>, <POINT (-69.925 -50.011)>]\n",
       "Length: 84812154, dtype: geometry"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "geometry"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d2e215d-0783-448e-be56-784567bc2eb1",
   "metadata": {
    "user_expressions": []
   },
   "source": [
    "#### Create `geopandas.GeoDataFrame` with h_ph and geometry columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "04a6040c-aea1-4425-84b1-2b4fc59f7afc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>h_ph</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>267.050385</td>\n",
       "      <td>POINT (-60.05933 -79.00597)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>279.333771</td>\n",
       "      <td>POINT (-60.05933 -79.00597)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>272.782135</td>\n",
       "      <td>POINT (-60.05933 -79.00597)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>226.964447</td>\n",
       "      <td>POINT (-60.05935 -79.00597)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>228.495010</td>\n",
       "      <td>POINT (-60.05935 -79.00597)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84812149</th>\n",
       "      <td>286.131195</td>\n",
       "      <td>POINT (-69.92508 -50.01094)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84812150</th>\n",
       "      <td>282.127655</td>\n",
       "      <td>POINT (-69.92508 -50.01094)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84812151</th>\n",
       "      <td>278.603241</td>\n",
       "      <td>POINT (-69.92508 -50.01094)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84812152</th>\n",
       "      <td>155.884216</td>\n",
       "      <td>POINT (-69.92513 -50.01094)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84812153</th>\n",
       "      <td>151.234497</td>\n",
       "      <td>POINT (-69.92514 -50.01094)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>84812154 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                h_ph                     geometry\n",
       "0         267.050385  POINT (-60.05933 -79.00597)\n",
       "1         279.333771  POINT (-60.05933 -79.00597)\n",
       "2         272.782135  POINT (-60.05933 -79.00597)\n",
       "3         226.964447  POINT (-60.05935 -79.00597)\n",
       "4         228.495010  POINT (-60.05935 -79.00597)\n",
       "...              ...                          ...\n",
       "84812149  286.131195  POINT (-69.92508 -50.01094)\n",
       "84812150  282.127655  POINT (-69.92508 -50.01094)\n",
       "84812151  278.603241  POINT (-69.92508 -50.01094)\n",
       "84812152  155.884216  POINT (-69.92513 -50.01094)\n",
       "84812153  151.234497  POINT (-69.92514 -50.01094)\n",
       "\n",
       "[84812154 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf = gpd.GeoDataFrame(data=df[[\"h_ph\"]], geometry=geometry)\n",
    "gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "43288ec0-38b7-470c-b4de-47d7a28bb9e1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Geographic 2D CRS: OGC:CRS84>\n",
       "Name: WGS 84 (CRS84)\n",
       "Axis Info [ellipsoidal]:\n",
       "- Lon[east]: Geodetic longitude (degree)\n",
       "- Lat[north]: Geodetic latitude (degree)\n",
       "Area of Use:\n",
       "- name: World.\n",
       "- bounds: (-180.0, -90.0, 180.0, 90.0)\n",
       "Datum: World Geodetic System 1984 ensemble\n",
       "- Ellipsoid: WGS 84\n",
       "- Prime Meridian: Greenwich"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.crs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ee22f0b-7912-4b6b-8d0c-fb5a629618b5",
   "metadata": {
    "user_expressions": []
   },
   "source": [
    "### Store as FlatGeobuf\n",
    "\n",
    "To save time, we will save the dataframe without a spatial index.\n",
    "This should take about 3min, compared to >1hour with a spatial index.\n",
    "\n",
    "References:\n",
    "- https://gdal.org/drivers/vector/flatgeobuf.html#layer-creation-options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1f71df6c-0a83-4ef9-b636-7f1ae2fcdec8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ATL03_20230211164520_08111812_006_01_gt1l_heights_h_ph.fgb'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "granule_name:str = os.path.basename(os.path.splitext(s3url)[0])\n",
    "filename:str = f\"{granule_name}_gt1l_heights_h_ph.fgb\"\n",
    "filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "29bff8b8-a263-40aa-ac32-225b237cfd31",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "gdf.to_file(filename=filename, driver=\"FlatGeobuf\", engine=\"pyogrio\", SPATIAL_INDEX=\"NO\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "365682bd-e2ae-43f4-976c-17143cbdd0bd",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "#### Add spatial index (optional)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9754bf6d-d5cf-4aa0-bcca-41c78c8ca365",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.44 s, sys: 682 ms, total: 2.12 s\n",
      "Wall time: 3min 43s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!ogr2ogr -f FlatGeobuf ATL03_20181120182818_08110112_006_02.fgb ATL03_20181120182818_08110112_006_02_gt1l_heights_h_ph.fgb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e00c90f-4ba8-4c06-a2d5-9739430ae37e",
   "metadata": {},
   "source": [
    "#### Check that FlatGeobuf can be loaded back into `geopandas.GeoDataFrame`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0747e693-4952-47c7-940d-3e0b315e2cd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 31s, sys: 9.96 s, total: 1min 41s\n",
      "Wall time: 1min 43s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_gdf = gpd.read_file(filename=filename, engine=\"pyogrio\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fb4b2d6-bdca-4a57-8e69-b26cc0105aa7",
   "metadata": {
    "user_expressions": []
   },
   "source": [
    "#### Upload FlatGeobuf to s3 bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c9302155-c697-499a-8e1c-e60cf99f0c20",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !aws s3 cp ATL03_20230211164520_08111812_006_01_gt1l_heights_h_ph.fgb s3://nasa-cryo-scratch/h5cloud/flatgeobuf/ATL03_20230211164520_08111812_006_01.fgb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d581357c-51b1-424c-a401-4f79e3f67775",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "## Multiple file conversions\n",
    "\n",
    "Convert many HDF5 files to FlatGeobuf in a for-loop,\n",
    "and upload them to an s3 bucket."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "61dd2a6c-9a03-40e5-b424-a61c1c456287",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def generate_flatgeobuf(hdf5_path: str) -> str:\n",
    "    # Read into H5DataFrame\n",
    "    h5 = h5py.File(name=s3.open(path=hdf5_path, mode=\"rb\"))\n",
    "    df = H5DataFrame(group=h5[\"gt1l/heights\"])\n",
    "    \n",
    "    # Create geopandas.GeoDataFrame\n",
    "    geometry = gpd.points_from_xy(x=df[\"lon_ph\"], y=df[\"lat_ph\"], crs=\"OGC:CRS84\")\n",
    "    gdf = gpd.GeoDataFrame(data=df[[\"h_ph\"]], geometry=geometry)\n",
    "    \n",
    "    # Save to FlatGeobuf\n",
    "    granule_name:str = os.path.basename(os.path.splitext(s3url)[0])\n",
    "    filename:str = f\"{granule_name}.fgb\"\n",
    "    gdf.to_file(filename=filename, driver=\"FlatGeobuf\", engine=\"pyogrio\", SPATIAL_INDEX=\"NO\")\n",
    "    \n",
    "    return filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7d54c4cb-c923-48f7-9f9e-2bc70b1ccb51",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [29:23<00:00, 352.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ATL03_20181120182818_08110112_006_02.fgb', 'ATL03_20190219140808_08110212_006_02.fgb', 'ATL03_20200217204710_08110612_006_01.fgb', 'ATL03_20211114142614_08111312_006_01.fgb', 'ATL03_20230211164520_08111812_006_01.fgb']\n",
      "CPU times: user 22min 11s, sys: 1min 8s, total: 23min 20s\n",
      "Wall time: 29min 23s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "flatgeobufs = []\n",
    "for s3url in tqdm.tqdm(iterable=s3urls):\n",
    "    flatgeobuf:str = generate_flatgeobuf(hdf5_path=s3url)\n",
    "    flatgeobufs.append(flatgeobuf)\n",
    "print(flatgeobufs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ab01c65c-10f0-4d63-a6d2-ba8047c992f2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding spatial index to ATL03_20181120182818_08110112_006_02.fgb\n",
      "0...10...20...30...40...50...60...70...80...90...100 - done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "real\t4m1.905s\n",
      "user\t2m40.654s\n",
      "sys\t0m12.627s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding spatial index to ATL03_20190219140808_08110212_006_02.fgb\n",
      "0...10...20...30...40...50...60...70...80...90...100 - done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "real\t6m28.763s\n",
      "user\t4m26.447s\n",
      "sys\t0m18.805s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding spatial index to ATL03_20200217204710_08110612_006_01.fgb\n",
      "0...10...20...30...40...50...60...70...80...90...100 - done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "real\t3m40.568s\n",
      "user\t2m43.953s\n",
      "sys\t0m11.046s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding spatial index to ATL03_20211114142614_08111312_006_01.fgb\n",
      "0...10...20...30...40...50...60...70...80...90...100 - done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "real\t9m7.924s\n",
      "user\t6m50.077s\n",
      "sys\t0m24.373s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding spatial index to ATL03_20230211164520_08111812_006_01.fgb\n",
      "0...10...20...30...40...50...60...70...80...90...100 - done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "real\t7m4.553s\n",
      "user\t5m2.343s\n",
      "sys\t0m21.201s\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "mkdir -p sindex\n",
    "for fgb in $(ls *.fgb); do\n",
    "    echo \"Adding spatial index to $fgb\"\n",
    "    time ogr2ogr -f FlatGeobuf -progress sindex/$fgb $fgb\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b5af34ea-2ebd-4995-a35c-000b0a4346a0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: sindex/ATL03_20200217204710_08110612_006_01.fgb to s3://nasa-cryo-scratch/h5cloud/flatgeobuf/ATL03_20200217204710_08110612_006_01.fgb\n",
      "upload: sindex/ATL03_20181120182818_08110112_006_02.fgb to s3://nasa-cryo-scratch/h5cloud/flatgeobuf/ATL03_20181120182818_08110112_006_02.fgb\n",
      "upload: sindex/ATL03_20190219140808_08110212_006_02.fgb to s3://nasa-cryo-scratch/h5cloud/flatgeobuf/ATL03_20190219140808_08110212_006_02.fgb\n",
      "upload: sindex/ATL03_20230211164520_08111812_006_01.fgb to s3://nasa-cryo-scratch/h5cloud/flatgeobuf/ATL03_20230211164520_08111812_006_01.fgb\n",
      "upload: sindex/ATL03_20211114142614_08111312_006_01.fgb to s3://nasa-cryo-scratch/h5cloud/flatgeobuf/ATL03_20211114142614_08111312_006_01.fgb\n",
      "2023-08-11 19:52:47 5702149992 ATL03_20181120182818_08110112_006_02.fgb\n",
      "2023-08-11 19:52:47 9048510952 ATL03_20190219140808_08110212_006_02.fgb\n",
      "2023-08-11 19:52:47 5622152472 ATL03_20200217204710_08110612_006_01.fgb\n",
      "2023-08-11 19:52:47 10721045272 ATL03_20211114142614_08111312_006_01.fgb\n",
      "2023-08-11 19:52:47 10403625072 ATL03_20230211164520_08111812_006_01.fgb\n"
     ]
    }
   ],
   "source": [
    "!aws s3 cp ./sindex/ s3://nasa-cryo-scratch/h5cloud/flatgeobuf/ --recursive --exclude \"*\" --include \"*.fgb\"\n",
    "!aws s3 ls s3://nasa-cryo-scratch/h5cloud/flatgeobuf/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}