2_Leiden_SDMbench_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f39d77c2-da9c-42be-9bb2-d1e78b37628d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "from datetime import datetime\n",
    "from io import StringIO\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1a574e9f-1f69-40ad-bdf3-0101ed54d380",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata_file = \"downloads.tsv\"\n",
    "\n",
    "data_dir = Path(\"./data/sdmbench\")\n",
    "result_dir = Path(\"./results/sdmbench\")\n",
    "\n",
    "performance_metric_file = \"./results/runtime_memory.tsv\"\n",
    "\n",
    "seed = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a99e1b41-1522-4c87-a040-9944e6a370c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_dir.mkdir(exist_ok=True, parents=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d1bdca26-435f-41c1-b1e1-79096018fe02",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = pd.read_table(metadata_file, index_col=0)\n",
    "\n",
    "metadata = metadata.assign(path=lambda df: data_dir / df[\"url\"].str.split(\"/\").str[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ff57600-8f9a-4ec5-a90a-a5a9eb811b6e",
   "metadata": {},
   "source": [
    "# Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0f0ecb14-5c3f-4243-8e27-2d9f89c46a24",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_pcs = 30\n",
    "n_genes = 3_000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2a595349-e02b-4ca1-8fa1-39f18725106a",
   "metadata": {},
   "outputs": [],
   "source": [
    "spatial_weights_delaunay = {\n",
    "    \"osmFISH\": 0.8,\n",
    "    \"MERFISH\": 1,\n",
    "    \"BARISTAseq\": 1.2,\n",
    "    \"STARmap\": 1,\n",
    "    \"STARmap*\": 0.8,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "410383ef-c454-44df-a79e-e27c5ccd5398",
   "metadata": {},
   "outputs": [],
   "source": [
    "spatial_weights_kNN10 = {\n",
    "    \"osmFISH\": 1.2,\n",
    "    \"MERFISH\": 1.8,\n",
    "    \"BARISTAseq\": 1.8,\n",
    "    \"STARmap\": 1.6,\n",
    "    \"STARmap*\": 1.4,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f464c79c-ffe1-40c1-91fd-274d100197d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "spatial_weight_stereoseq = 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "62f367ba-0a8a-4b94-afc2-95511c63aa8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_seeds = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3bd0c0ef-5e25-4158-ac64-6d20c97202fa",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "conda_env = \"spatialleiden\"\n",
    "conda_path = \"~/miniconda3/bin/activate\"\n",
    "\n",
    "log_dir = result_dir / \"logs\"\n",
    "log_dir.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "conda_cmd = f\"source {conda_path} {conda_env}\"\n",
    "\n",
    "path = Path(os.getcwd())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e975712-1b08-4321-a13c-e26575ec6e1e",
   "metadata": {},
   "source": [
    "# Submit jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6e4b6ebd-ab65-4675-a3eb-95990b6ebfae",
   "metadata": {},
   "outputs": [],
   "source": [
    "start = datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ac4e1ce2-db77-45fc-afc4-1efd13c0a568",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_ids = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b049e0a0-783c-4b33-968f-4ca5221cae2e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Stereo-seq -> only sequencing-based method\n",
    "\n",
    "for sample in metadata.loc[lambda df: df[\"technology\"] == \"Stereo-seq\"].itertuples():\n",
    "    for seed in range(n_seeds):\n",
    "        name = f\"{sample.Index}_seed{seed}_neighbors_grid4\"\n",
    "        out = result_dir / f\"{name}.tsv\"\n",
    "        cmd = (\n",
    "            f\"{path/'benchmark.py'} \"\n",
    "            f\"-i {sample.path} \"\n",
    "            f\"-o {out} \"\n",
    "            f\"--spatial_weight {spatial_weight_stereoseq} \"\n",
    "            f\"--n_pcs {n_pcs} \"\n",
    "            f\"--n_genes {n_genes} \"\n",
    "            f\"--seed {seed} \"\n",
    "            \"--svg \"\n",
    "            \"--stereoseq \"\n",
    "        )\n",
    "\n",
    "        id_string = os.popen(\n",
    "            f\"sbatch -J {name} --mem=5G -n 8 -N 1 \"\n",
    "            f\"-o {log_dir/name}.txt \"\n",
    "            f'--wrap=\"{conda_cmd} && {cmd}\" '\n",
    "        ).read()\n",
    "        job_ids.append(id_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "590189c9-a432-4aaf-ae8a-84b6754aff27",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# all imaging-based methods\n",
    "\n",
    "for sample in metadata.loc[lambda df: df[\"technology\"] != \"Stereo-seq\"].itertuples():\n",
    "    for seed in range(5):\n",
    "        for neighbors in [\"delaunay\", 10]:\n",
    "            name = f\"{sample.Index}_seed{seed}_neighbors_{neighbors}\"\n",
    "            out = result_dir / f\"{name}.tsv\"\n",
    "            if neighbors == \"delaunay\":\n",
    "                w = spatial_weights_delaunay[sample.technology]\n",
    "            else:\n",
    "                w = spatial_weights_kNN10[sample.technology]\n",
    "            cmd = (\n",
    "                f\"{path/'benchmark.py'} \"\n",
    "                f\"-i {sample.path} \"\n",
    "                f\"-o {out} \"\n",
    "                f\"--spatial_weight {w} \"\n",
    "                f\"--n_pcs {n_pcs} \"\n",
    "                f\"--neighbors {neighbors} \"\n",
    "                f\"--seed {seed} \"\n",
    "                \"--log \"\n",
    "                \"--mspca \"\n",
    "            )\n",
    "\n",
    "            id_string = os.popen(\n",
    "                f\"sbatch -J {name} --mem=5G -n 8 -N 1 \"\n",
    "                f\"-o {log_dir/name}.txt \"\n",
    "                f'--wrap=\"{conda_cmd} && {cmd}\" '\n",
    "            ).read()\n",
    "            job_ids.append(id_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44743157-b18f-4c32-9ddb-54dec5477498",
   "metadata": {},
   "source": [
    "# Runtime metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8aeead1e-c2ed-4fa2-90f8-85e02432a1e5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def parse_job_id(bsub_out):\n",
    "    return int(re.search(\"(\\d+)$\", bsub_out).group(1))\n",
    "\n",
    "\n",
    "job_ids = [parse_job_id(j_id.strip()) for j_id in job_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "036b8f7a-7f4a-42ea-b1ea-533e0f4141ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_stats = os.popen(\n",
    "    (\n",
    "        \"sacct \"\n",
    "        f\"--starttime {start.strftime('%Y-%m-%d%H:%M:%S')} \"\n",
    "        \"--format='JobID,Jobname%50,TotalCPU,Elapsed,MaxRSS' \"\n",
    "        \"-P \"\n",
    "        \"--delimiter=$'\\t' \"\n",
    "        \"--units=M \"\n",
    "    )\n",
    ").read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b0462673-b76c-415b-9c6d-6773e58a24df",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_stats = pd.read_table(StringIO(job_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "20ae3c55-2269-4db7-a153-a9a5e354042d",
   "metadata": {},
   "outputs": [],
   "source": [
    "cpu_stats = (\n",
    "    job_stats.loc[\n",
    "        lambda df: ~df[\"JobID\"].str.contains(\".\", regex=False),\n",
    "        [\"JobID\", \"JobName\", \"TotalCPU\", \"Elapsed\"],\n",
    "    ]\n",
    "    .assign(\n",
    "        JobID=lambda df: df[\"JobID\"].astype(int),\n",
    "        TotalCPU=lambda df: (\"00:\" + df[\"TotalCPU\"]).str.extract(\n",
    "            \"(\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?)$\"\n",
    "        ),\n",
    "        Elapsed=lambda df: (\"00:\" + df[\"Elapsed\"]).str.extract(\n",
    "            \"(\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?)$\"\n",
    "        ),\n",
    "    )\n",
    "    .assign(\n",
    "        TotalCPU=lambda df: pd.to_timedelta(df[\"TotalCPU\"]).dt.total_seconds(),\n",
    "        Elapsed=lambda df: pd.to_timedelta(df[\"Elapsed\"]).dt.total_seconds(),\n",
    "    )\n",
    "    .set_index(\"JobID\")\n",
    "    .rename(columns={\"TotalCPU\": \"CPU time [s]\", \"Elapsed\": \"wall time [s]\"})\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "43b84780-7efc-477c-8f60-7e3c72d10a50",
   "metadata": {},
   "outputs": [],
   "source": [
    "memory_stats = (\n",
    "    job_stats.loc[\n",
    "        lambda df: df[\"JobID\"].str.contains(\".batch\", regex=False), [\"JobID\", \"MaxRSS\"]\n",
    "    ]\n",
    "    .assign(\n",
    "        JobID=lambda df: df[\"JobID\"].str.extract(\"(\\d+)\").astype(int),\n",
    "        MaxRSS=lambda df: df[\"MaxRSS\"].str.extract(\"([\\d\\\\.]+)\").astype(float),\n",
    "    )\n",
    "    .set_index(\"JobID\")\n",
    "    .rename(columns={\"MaxRSS\": \"max memory [MB]\"})\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "91682a90-3bca-4378-99f2-92bebd7d0d45",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = cpu_stats.join(memory_stats).loc[lambda df: df.index.isin(job_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0c4e2c3a-00ae-42b1-9213-a861de59031b",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats.to_csv(performance_metric_file, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9c35553-6232-46a4-8105-af670484ef07",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:muellni-spatialleiden]",
   "language": "python",
   "name": "conda-env-muellni-spatialleiden-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}