mimic-iv-cxr/notebooks/generate-stats-from-paper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import os\n",
    "from collections import OrderedDict\n",
    "from pathlib import Path\n",
    "import json\n",
    "import gzip\n",
    "\n",
    "\n",
    "import pydicom\n",
    "from pydicom.tag import Tag\n",
    "from pydicom._dicom_dict import DicomDictionary\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summarize dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "377110 DICOMs in MIMIC-CXR v2.0.0.\n",
      "  227835 studies.\n",
      "  65379 subjects.\n"
     ]
    }
   ],
   "source": [
    "# load in mapping file\n",
    "mimic_cxr_path = Path('/db/mimic-cxr')\n",
    "df = pd.read_csv(mimic_cxr_path / 'cxr-record-list.csv.gz', header=0, sep=',')\n",
    "\n",
    "n = df.shape[0]\n",
    "print(f'{n} DICOMs in MIMIC-CXR v2.0.0.')\n",
    "\n",
    "n = df['study_id'].nunique()\n",
    "print(f'  {n} studies.')\n",
    "\n",
    "n = df['subject_id'].nunique()\n",
    "print(f'  {n} subjects.')\n",
    "\n",
    "dicoms = set(df['dicom_id'].tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_split = pd.read_csv(mimic_cxr_path / 'mimic-cxr-2.0.0-split.csv.gz')\n",
    "df_metadata = pd.read_csv(mimic_cxr_path / 'mimic-cxr-2.0.0-metadata.csv.gz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize view with a mapping from ViewPosition\n",
    "VIEW_MAP = {\n",
    "    'AP': 'frontal',\n",
    "    'PA': 'frontal',\n",
    "    'LATERAL': 'lateral',\n",
    "    'LL': 'lateral',\n",
    "    'LPO': 'other',\n",
    "    'RAO': 'other',\n",
    "    'RPO': 'other',\n",
    "    'LAO': 'other',\n",
    "    # the below are overwritten in some instances by manual review\n",
    "    'AP AXIAL': 'other',\n",
    "    'XTABLE LATERAL': 'other',\n",
    "    'AP LLD': 'other',\n",
    "    'PA LLD': 'other',\n",
    "    'L5 S1': 'other',\n",
    "    'SWIMMERS': 'other',\n",
    "    'AP RLD': 'other',\n",
    "    'PA RLD': 'other',\n",
    "}\n",
    "\n",
    "df_metadata['view'] = df_metadata['ViewPosition'].map(VIEW_MAP)\n",
    "\n",
    "# for 'other' category, currently many of these are simply unknown\n",
    "# so try to update them with acq device map\n",
    "ADPD_MAP = {\n",
    "    'CHEST, LATERAL': 'lateral',\n",
    "    'CHEST, PA': 'frontal',\n",
    "    # manually checked 100 records, below is always frontal\n",
    "    'CHEST, PORTABLE': 'frontal',\n",
    "    'CHEST, PA X-WISE': 'frontal',\n",
    "    'CHEST, AP (GRID)': 'frontal',\n",
    "    'CHEST LAT': 'lateral',\n",
    "    'CHEST PA': 'frontal',\n",
    "    'CHEST, AP NON-GRID': 'frontal',\n",
    "    'CHEST AP NON GRID': 'frontal',\n",
    "    'CHEST PA X-WISE': 'frontal',\n",
    "    'CHEST AP GRID': 'frontal',\n",
    "    'CHEST, PORTABLE X-WISE': 'other',\n",
    "    # below have < 25 samples each\n",
    "    'CHEST PORT': 'frontal',\n",
    "    'CHEST PORT X-WISE': 'frontal',\n",
    "    # manually classified below\n",
    "    'SHOULDER': 'other',\n",
    "    'CHEST, PEDI (4-10 YRS)': 'other',\n",
    "    'LOWER RIBS': 'other',\n",
    "    'CHEST, DECUB.': 'other',\n",
    "    'ABDOMEN, PORTABLE': 'other',\n",
    "    'UPPER RIBS': 'frontal',\n",
    "    'STERNUM, LATERAL': 'lateral',\n",
    "    'KNEE, AP/OBL': 'other',\n",
    "    'STERNUM, PA/OBL.': 'other',\n",
    "    'CLAVICLE/ AC JOINTS': 'other',\n",
    "    'ABDOMEN,GENERAL': 'other',\n",
    "    'LOWER RIB': 'other',\n",
    "    'SCOLIOSIS AP': 'frontal'\n",
    "}\n",
    "\n",
    "good_view = ['frontal', 'lateral']\n",
    "idxUpdate = ~df_metadata['view'].isin(good_view)\n",
    "c = 'AcquisitionDeviceProcessingDescription'\n",
    "idx = (df_metadata[c].notnull()) & idxUpdate\n",
    "df_metadata.loc[idx, 'view'] = df_metadata.loc[idx, c].map(ADPD_MAP)\n",
    "\n",
    "DICOM_TO_VIEW = {\n",
    "    '2164992c-f4abb30a-7aaaf4f4-383cab47-4e3eb1c8': ['PA', 'frontal'],\n",
    "    '5e6881e2-ff4254e0-b99f0c2f-8964482a-031364db': ['LL', 'lateral'],\n",
    "    'fcdf7a30-3236b74e-65b97587-cdd4cfde-63cd1de0': ['PA', 'frontal'],\n",
    "    'fb074ec1-6715839c-84fa75e6-adc3f026-448b1481': ['PA', 'frontal'],\n",
    "    'dfb8080a-8506e43e-840d9d58-0f738f41-82c120b0': ['PA', 'frontal'],\n",
    "    '4b32608b-c2ead7c4-1fe5565f-42f7ab80-9dad30de': ['LL', 'lateral'],\n",
    "    '53663e89-8f9ca9bb-df1bf434-8d6b1283-2b612609': ['LL', 'lateral'],\n",
    "    # below are AP, but incorrectly in View Position\n",
    "    '8672a4e7-366801a0-26cf2395-9344335c-aac8d728': ['AP', 'frontal'],\n",
    "    '9800b28e-3ff3b417-18473be2-1a66131d-aca88488': ['AP', 'frontal'],\n",
    "    '598cfe48-33a8643e-843e27e2-5dd584e7-3cd5f1c0': ['AP', 'frontal']\n",
    "}\n",
    "\n",
    "# we manually reviewed a few DICOMs to keep them in\n",
    "for dcm, row in DICOM_TO_VIEW.items():\n",
    "    view = row[1]\n",
    "    idx = df_metadata['dicom_id'] == dcm\n",
    "    if idx.any():\n",
    "        df_metadata.loc[idx, 'view'] = view"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>dicom_id</th>\n",
       "      <td>02aa804e-bde0afdd-112c0b34-7bc16630-4e384014</td>\n",
       "      <td>174413ec-4ec4c1f7-34ea26b7-c5f994f8-79ef1962</td>\n",
       "      <td>2a2277a9-b0ded155-c0de8eb9-c124d10e-82c5caab</td>\n",
       "      <td>e084de3b-be89b11e-20fe3f9f-9c8d8dfe-4cfd202c</td>\n",
       "      <td>68b5c4b1-227d0485-9cc38c3f-7b84ab51-4b472714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>split</th>\n",
       "      <td>train</td>\n",
       "      <td>train</td>\n",
       "      <td>train</td>\n",
       "      <td>train</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>view</th>\n",
       "      <td>frontal</td>\n",
       "      <td>lateral</td>\n",
       "      <td>frontal</td>\n",
       "      <td>lateral</td>\n",
       "      <td>frontal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Atelectasis</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cardiomegaly</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Consolidation</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edema</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Enlarged Cardiomediastinum</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fracture</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Lesion</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Opacity</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No Finding</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Effusion</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Other</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumonia</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumothorax</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Support Devices</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                       0  \\\n",
       "dicom_id                    02aa804e-bde0afdd-112c0b34-7bc16630-4e384014   \n",
       "split                                                              train   \n",
       "view                                                             frontal   \n",
       "Atelectasis                                                          NaN   \n",
       "Cardiomegaly                                                         NaN   \n",
       "Consolidation                                                        NaN   \n",
       "Edema                                                                NaN   \n",
       "Enlarged Cardiomediastinum                                           NaN   \n",
       "Fracture                                                             NaN   \n",
       "Lung Lesion                                                          NaN   \n",
       "Lung Opacity                                                         NaN   \n",
       "No Finding                                                             1   \n",
       "Pleural Effusion                                                     NaN   \n",
       "Pleural Other                                                        NaN   \n",
       "Pneumonia                                                            NaN   \n",
       "Pneumothorax                                                         NaN   \n",
       "Support Devices                                                      NaN   \n",
       "\n",
       "                                                                       1  \\\n",
       "dicom_id                    174413ec-4ec4c1f7-34ea26b7-c5f994f8-79ef1962   \n",
       "split                                                              train   \n",
       "view                                                             lateral   \n",
       "Atelectasis                                                          NaN   \n",
       "Cardiomegaly                                                         NaN   \n",
       "Consolidation                                                        NaN   \n",
       "Edema                                                                NaN   \n",
       "Enlarged Cardiomediastinum                                           NaN   \n",
       "Fracture                                                             NaN   \n",
       "Lung Lesion                                                          NaN   \n",
       "Lung Opacity                                                         NaN   \n",
       "No Finding                                                             1   \n",
       "Pleural Effusion                                                     NaN   \n",
       "Pleural Other                                                        NaN   \n",
       "Pneumonia                                                            NaN   \n",
       "Pneumothorax                                                         NaN   \n",
       "Support Devices                                                      NaN   \n",
       "\n",
       "                                                                       2  \\\n",
       "dicom_id                    2a2277a9-b0ded155-c0de8eb9-c124d10e-82c5caab   \n",
       "split                                                              train   \n",
       "view                                                             frontal   \n",
       "Atelectasis                                                          NaN   \n",
       "Cardiomegaly                                                         NaN   \n",
       "Consolidation                                                        NaN   \n",
       "Edema                                                                NaN   \n",
       "Enlarged Cardiomediastinum                                           NaN   \n",
       "Fracture                                                             NaN   \n",
       "Lung Lesion                                                          NaN   \n",
       "Lung Opacity                                                         NaN   \n",
       "No Finding                                                             1   \n",
       "Pleural Effusion                                                     NaN   \n",
       "Pleural Other                                                        NaN   \n",
       "Pneumonia                                                            NaN   \n",
       "Pneumothorax                                                         NaN   \n",
       "Support Devices                                                      NaN   \n",
       "\n",
       "                                                                       3  \\\n",
       "dicom_id                    e084de3b-be89b11e-20fe3f9f-9c8d8dfe-4cfd202c   \n",
       "split                                                              train   \n",
       "view                                                             lateral   \n",
       "Atelectasis                                                          NaN   \n",
       "Cardiomegaly                                                         NaN   \n",
       "Consolidation                                                        NaN   \n",
       "Edema                                                                NaN   \n",
       "Enlarged Cardiomediastinum                                           NaN   \n",
       "Fracture                                                             NaN   \n",
       "Lung Lesion                                                          NaN   \n",
       "Lung Opacity                                                         NaN   \n",
       "No Finding                                                             1   \n",
       "Pleural Effusion                                                     NaN   \n",
       "Pleural Other                                                        NaN   \n",
       "Pneumonia                                                            NaN   \n",
       "Pneumothorax                                                         NaN   \n",
       "Support Devices                                                      NaN   \n",
       "\n",
       "                                                                       4  \n",
       "dicom_id                    68b5c4b1-227d0485-9cc38c3f-7b84ab51-4b472714  \n",
       "split                                                              train  \n",
       "view                                                             frontal  \n",
       "Atelectasis                                                          NaN  \n",
       "Cardiomegaly                                                         NaN  \n",
       "Consolidation                                                        NaN  \n",
       "Edema                                                                NaN  \n",
       "Enlarged Cardiomediastinum                                           NaN  \n",
       "Fracture                                                             NaN  \n",
       "Lung Lesion                                                          NaN  \n",
       "Lung Opacity                                                         NaN  \n",
       "No Finding                                                             1  \n",
       "Pleural Effusion                                                     NaN  \n",
       "Pleural Other                                                        NaN  \n",
       "Pneumonia                                                            NaN  \n",
       "Pneumothorax                                                         NaN  \n",
       "Support Devices                                                      NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_split.merge(df_metadata.drop(['study_id', 'subject_id'], axis=1),\n",
    "                   on='dicom_id', how='inner')\n",
    "\n",
    "\n",
    "nb = pd.read_csv(mimic_cxr_path / 'mimic-cxr-2.0.0-negbio.csv.gz')\n",
    "# avoid redundant columns\n",
    "nb.drop('subject_id', axis=1, inplace=True)\n",
    "findings = [x for x in nb.columns if x != 'study_id']\n",
    "df = df.merge(nb, how='left', on='study_id')\n",
    "\n",
    "# indicator flag for the study having a NegBio finding\n",
    "df['has_negbio_finding'] = df[[x for x in findings if x != 'No Finding']].notnull().sum(axis=1) > 0\n",
    "\n",
    "df[['dicom_id', 'split', 'view'] + findings].head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train</th>\n",
       "      <th>validate</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Number of images</th>\n",
       "      <td>368960</td>\n",
       "      <td>2991</td>\n",
       "      <td>5159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>frontal</th>\n",
       "      <td>248020 (67.2%)</td>\n",
       "      <td>2041 (68.2%)</td>\n",
       "      <td>3653 (70.8%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lateral</th>\n",
       "      <td>120795 (32.7%)</td>\n",
       "      <td>949 (31.7%)</td>\n",
       "      <td>1502 (29.1%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>145 (0.0%)</td>\n",
       "      <td>1 (0.0%)</td>\n",
       "      <td>4 (0.1%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Number of studies</th>\n",
       "      <td>222758</td>\n",
       "      <td>1808</td>\n",
       "      <td>3269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>with a finding</th>\n",
       "      <td>170420 (76.5%)</td>\n",
       "      <td>1394 (77.1%)</td>\n",
       "      <td>2912 (89.1%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Number of patients</th>\n",
       "      <td>64586</td>\n",
       "      <td>500</td>\n",
       "      <td>293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>with a finding</th>\n",
       "      <td>44157 (68.4%)</td>\n",
       "      <td>344 (68.8%)</td>\n",
       "      <td>288 (98.3%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             train      validate          test\n",
       "Number of images            368960          2991          5159\n",
       "frontal             248020 (67.2%)  2041 (68.2%)  3653 (70.8%)\n",
       "lateral             120795 (32.7%)   949 (31.7%)  1502 (29.1%)\n",
       "other                   145 (0.0%)      1 (0.0%)      4 (0.1%)\n",
       "Number of studies           222758          1808          3269\n",
       "  with a finding    170420 (76.5%)  1394 (77.1%)  2912 (89.1%)\n",
       "Number of patients           64586           500           293\n",
       "  with a finding     44157 (68.4%)   344 (68.8%)   288 (98.3%)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits = ['train', 'validate', 'test']\n",
    "split_views = df.groupby(['split', 'view'])[['dicom_id']].count()\n",
    "\n",
    "row_idx = ['frontal', 'lateral', 'other']\n",
    "\n",
    "# number of images in each set\n",
    "n_images = {}\n",
    "for c in splits:\n",
    "    n_images[c] = split_views.loc[c].loc[row_idx, 'dicom_id'].sum()\n",
    "\n",
    "tbl = pd.DataFrame.from_dict(n_images, orient='index')\n",
    "tbl.columns = ['Number of images']\n",
    "tbl = tbl.T\n",
    "\n",
    "# number of images in each set for each view\n",
    "n_images = {}\n",
    "for c in splits:\n",
    "    n_images[c] = {}\n",
    "    for view in row_idx:\n",
    "        n_images[c][view] = split_views.loc[c].loc[view, 'dicom_id']\n",
    "n_images = pd.DataFrame.from_dict(n_images, orient='index')\n",
    "n_images = n_images.T\n",
    "\n",
    "\n",
    "# convert frontal/lateral/other into \"N (%)\"\n",
    "for i in n_images.index:\n",
    "    for c in splits:\n",
    "        val = n_images.loc[i, c]\n",
    "        n_images.loc[i, c] = f'{val} ({100.0*val/tbl.loc[\"Number of images\", c]:3.1f}%)'\n",
    "\n",
    "tbl = pd.concat([tbl, n_images], axis=0, sort=False)\n",
    "\n",
    "# add in the number of subjects\n",
    "n_studies = df.groupby('split')[['study_id']].nunique().T\n",
    "n_studies.index = ['Number of studies']\n",
    "tbl = pd.concat([tbl, n_studies], axis=0, sort=False)\n",
    "\n",
    "# studies with a finding\n",
    "n_studies = df.loc[df['has_negbio_finding']].groupby('split')[['study_id']].nunique().T\n",
    "n_studies.index = ['  with a finding']\n",
    "for c in splits:\n",
    "    val = n_studies.loc['  with a finding', c]\n",
    "    n_studies.loc['  with a finding', c] = f'{val} ({100.0*val/tbl.loc[\"Number of studies\", c]:3.1f}%)'\n",
    "tbl = pd.concat([tbl, n_studies], axis=0, sort=False)\n",
    "\n",
    "# patients\n",
    "n_pt = df.groupby('split')[['subject_id']].nunique().T\n",
    "n_pt.index = ['Number of patients']\n",
    "tbl = pd.concat([tbl, n_pt], axis=0, sort=False)\n",
    "\n",
    "\n",
    "# patients with a finding\n",
    "n_studies = df.loc[df['has_negbio_finding']].groupby('split')[['subject_id']].nunique().T\n",
    "n_studies.index = ['  with a finding']\n",
    "for c in splits:\n",
    "    val = n_studies.loc['  with a finding', c]\n",
    "    n_studies.loc['  with a finding', c] = f'{val} ({100.0*val/tbl.loc[\"Number of patients\", c]:3.1f}%)'\n",
    "tbl = pd.concat([tbl, n_studies], axis=0, sort=False)\n",
    "\n",
    "tbl.to_latex('table2.tex')\n",
    "\n",
    "tbl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Frequency of findings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subject_id</th>\n",
       "      <th>study_id</th>\n",
       "      <th>Atelectasis</th>\n",
       "      <th>Cardiomegaly</th>\n",
       "      <th>Consolidation</th>\n",
       "      <th>Edema</th>\n",
       "      <th>Enlarged Cardiomediastinum</th>\n",
       "      <th>Fracture</th>\n",
       "      <th>Lung Lesion</th>\n",
       "      <th>Lung Opacity</th>\n",
       "      <th>No Finding</th>\n",
       "      <th>Pleural Effusion</th>\n",
       "      <th>Pleural Other</th>\n",
       "      <th>Pneumonia</th>\n",
       "      <th>Pneumothorax</th>\n",
       "      <th>Support Devices</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10000032</td>\n",
       "      <td>50414267</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10000032</td>\n",
       "      <td>53189527</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10000032</td>\n",
       "      <td>53911762</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10000032</td>\n",
       "      <td>56699142</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10000764</td>\n",
       "      <td>57375967</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Uncertain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>10000898</td>\n",
       "      <td>50771383</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10000898</td>\n",
       "      <td>54205396</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10000935</td>\n",
       "      <td>50578979</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Uncertain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Disagreement</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>10000935</td>\n",
       "      <td>51178377</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Uncertain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10000935</td>\n",
       "      <td>55697293</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Positive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   subject_id  study_id Atelectasis Cardiomegaly Consolidation      Edema  \\\n",
       "0    10000032  50414267         NaN          NaN           NaN        NaN   \n",
       "1    10000032  53189527         NaN          NaN           NaN        NaN   \n",
       "2    10000032  53911762         NaN          NaN           NaN        NaN   \n",
       "3    10000032  56699142         NaN          NaN           NaN        NaN   \n",
       "4    10000764  57375967         NaN          NaN      Positive        NaN   \n",
       "5    10000898  50771383         NaN          NaN           NaN        NaN   \n",
       "6    10000898  54205396         NaN          NaN           NaN        NaN   \n",
       "7    10000935  50578979         NaN          NaN           NaN  Uncertain   \n",
       "8    10000935  51178377         NaN          NaN           NaN        NaN   \n",
       "9    10000935  55697293         NaN          NaN           NaN        NaN   \n",
       "\n",
       "  Enlarged Cardiomediastinum Fracture Lung Lesion  Lung Opacity No Finding  \\\n",
       "0                        NaN      NaN         NaN           NaN   Positive   \n",
       "1                        NaN      NaN         NaN           NaN   Positive   \n",
       "2                        NaN      NaN         NaN           NaN   Positive   \n",
       "3                        NaN      NaN         NaN           NaN   Positive   \n",
       "4                        NaN      NaN         NaN           NaN        NaN   \n",
       "5                        NaN      NaN         NaN           NaN   Positive   \n",
       "6                        NaN      NaN         NaN           NaN   Positive   \n",
       "7                        NaN      NaN         NaN  Disagreement        NaN   \n",
       "8                        NaN      NaN         NaN      Positive        NaN   \n",
       "9                        NaN      NaN         NaN           NaN   Positive   \n",
       "\n",
       "  Pleural Effusion Pleural Other  Pneumonia Pneumothorax Support Devices  \n",
       "0              NaN           NaN        NaN          NaN             NaN  \n",
       "1              NaN           NaN        NaN          NaN             NaN  \n",
       "2              NaN           NaN        NaN          NaN             NaN  \n",
       "3              NaN           NaN        NaN          NaN             NaN  \n",
       "4              NaN           NaN  Uncertain          NaN             NaN  \n",
       "5              NaN           NaN        NaN          NaN             NaN  \n",
       "6              NaN           NaN        NaN          NaN             NaN  \n",
       "7         Positive           NaN   Positive          NaN             NaN  \n",
       "8              NaN           NaN  Uncertain          NaN             NaN  \n",
       "9              NaN           NaN        NaN          NaN             NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frequency of labels in MIMIC-CXR-JPG on the training subset of 222,750 unique radiologic studies.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Positive</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Uncertain</th>\n",
       "      <th>Disagreement</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Atelectasis</th>\n",
       "      <td>44,012 (19.8%)</td>\n",
       "      <td>921.0 (0.4%)</td>\n",
       "      <td>9,623.0 (4.3%)</td>\n",
       "      <td>1,705 (0.8%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cardiomegaly</th>\n",
       "      <td>38,002 (17.1%)</td>\n",
       "      <td>15,563.0 (7.0%)</td>\n",
       "      <td>5,753.0 (2.6%)</td>\n",
       "      <td>5,769 (2.6%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Consolidation</th>\n",
       "      <td>10,199 (4.6%)</td>\n",
       "      <td>7,791.0 (3.5%)</td>\n",
       "      <td>2,913.0 (1.3%)</td>\n",
       "      <td>1,576 (0.7%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edema</th>\n",
       "      <td>25,549 (11.5%)</td>\n",
       "      <td>24,746.0 (11.1%)</td>\n",
       "      <td>11,426.0 (5.1%)</td>\n",
       "      <td>2,282 (1.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Enlarged Cardiomediastinum</th>\n",
       "      <td>6,798 (3.1%)</td>\n",
       "      <td>5,158.0 (2.3%)</td>\n",
       "      <td>9,015.0 (4.0%)</td>\n",
       "      <td>248 (0.1%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fracture</th>\n",
       "      <td>3,675 (1.6%)</td>\n",
       "      <td>871.0 (0.4%)</td>\n",
       "      <td>295.0 (0.1%)</td>\n",
       "      <td>867 (0.4%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Lesion</th>\n",
       "      <td>5,939 (2.7%)</td>\n",
       "      <td>822.0 (0.4%)</td>\n",
       "      <td>996.0 (0.4%)</td>\n",
       "      <td>289 (0.1%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lung Opacity</th>\n",
       "      <td>49,512 (22.2%)</td>\n",
       "      <td>2,794.0 (1.3%)</td>\n",
       "      <td>2,052.0 (0.9%)</td>\n",
       "      <td>2,460 (1.1%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No Finding</th>\n",
       "      <td>74,019 (33.2%)</td>\n",
       "      <td>nan (nan%)</td>\n",
       "      <td>nan (nan%)</td>\n",
       "      <td>3,825 (1.7%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Effusion</th>\n",
       "      <td>51,680 (23.2%)</td>\n",
       "      <td>26,532.0 (11.9%)</td>\n",
       "      <td>5,184.0 (2.3%)</td>\n",
       "      <td>1,617 (0.7%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pleural Other</th>\n",
       "      <td>1,884 (0.8%)</td>\n",
       "      <td>120.0 (0.1%)</td>\n",
       "      <td>707.0 (0.3%)</td>\n",
       "      <td>91 (0.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumonia</th>\n",
       "      <td>15,333 (6.9%)</td>\n",
       "      <td>23,771.0 (10.7%)</td>\n",
       "      <td>17,313.0 (7.8%)</td>\n",
       "      <td>1,377 (0.6%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pneumothorax</th>\n",
       "      <td>9,159 (4.1%)</td>\n",
       "      <td>41,250.0 (18.5%)</td>\n",
       "      <td>848.0 (0.4%)</td>\n",
       "      <td>1,291 (0.6%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Support Devices</th>\n",
       "      <td>63,971 (28.7%)</td>\n",
       "      <td>3,014.0 (1.4%)</td>\n",
       "      <td>95.0 (0.0%)</td>\n",
       "      <td>1,785 (0.8%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Positive          Negative        Uncertain  \\\n",
       "Atelectasis                 44,012 (19.8%)      921.0 (0.4%)   9,623.0 (4.3%)   \n",
       "Cardiomegaly                38,002 (17.1%)   15,563.0 (7.0%)   5,753.0 (2.6%)   \n",
       "Consolidation                10,199 (4.6%)    7,791.0 (3.5%)   2,913.0 (1.3%)   \n",
       "Edema                       25,549 (11.5%)  24,746.0 (11.1%)  11,426.0 (5.1%)   \n",
       "Enlarged Cardiomediastinum    6,798 (3.1%)    5,158.0 (2.3%)   9,015.0 (4.0%)   \n",
       "Fracture                      3,675 (1.6%)      871.0 (0.4%)     295.0 (0.1%)   \n",
       "Lung Lesion                   5,939 (2.7%)      822.0 (0.4%)     996.0 (0.4%)   \n",
       "Lung Opacity                49,512 (22.2%)    2,794.0 (1.3%)   2,052.0 (0.9%)   \n",
       "No Finding                  74,019 (33.2%)        nan (nan%)       nan (nan%)   \n",
       "Pleural Effusion            51,680 (23.2%)  26,532.0 (11.9%)   5,184.0 (2.3%)   \n",
       "Pleural Other                 1,884 (0.8%)      120.0 (0.1%)     707.0 (0.3%)   \n",
       "Pneumonia                    15,333 (6.9%)  23,771.0 (10.7%)  17,313.0 (7.8%)   \n",
       "Pneumothorax                  9,159 (4.1%)  41,250.0 (18.5%)     848.0 (0.4%)   \n",
       "Support Devices             63,971 (28.7%)    3,014.0 (1.4%)      95.0 (0.0%)   \n",
       "\n",
       "                            Disagreement  \n",
       "Atelectasis                 1,705 (0.8%)  \n",
       "Cardiomegaly                5,769 (2.6%)  \n",
       "Consolidation               1,576 (0.7%)  \n",
       "Edema                       2,282 (1.0%)  \n",
       "Enlarged Cardiomediastinum    248 (0.1%)  \n",
       "Fracture                      867 (0.4%)  \n",
       "Lung Lesion                   289 (0.1%)  \n",
       "Lung Opacity                2,460 (1.1%)  \n",
       "No Finding                  3,825 (1.7%)  \n",
       "Pleural Effusion            1,617 (0.7%)  \n",
       "Pleural Other                  91 (0.0%)  \n",
       "Pneumonia                   1,377 (0.6%)  \n",
       "Pneumothorax                1,291 (0.6%)  \n",
       "Support Devices             1,785 (0.8%)  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb = pd.read_csv(mimic_cxr_path / 'mimic-cxr-2.0.0-negbio.csv.gz')\n",
    "cx = pd.read_csv(mimic_cxr_path / 'mimic-cxr-2.0.0-chexpert.csv.gz')\n",
    "\n",
    "# merge these findings to create a table\n",
    "# both agree -> output label\n",
    "# disagree -> output -9\n",
    "\n",
    "# drop subject_id from cx - we have it in nb\n",
    "df = nb.merge(\n",
    "    cx.drop('subject_id', axis=1),\n",
    "    how='left',\n",
    "    left_on='study_id', right_on='study_id',\n",
    "    suffixes=('', '_cx')\n",
    ")\n",
    "\n",
    "# subselect to training set\n",
    "study_ids = set(df_split.loc[df_split['split']=='train', 'study_id'])\n",
    "df = df.loc[df['study_id'].isin(study_ids)]\n",
    "\n",
    "# replace numeric labels with meaningful labels\n",
    "# also annotate disagreements between the two labelers\n",
    "labels = {0: 'Negative', 1: 'Positive', -1: 'Uncertain', -9: 'Disagreement'}\n",
    "for c in df.columns:\n",
    "    if c in ('subject_id', 'study_id'):\n",
    "        continue\n",
    "    elif c.endswith('_cx'):\n",
    "        continue\n",
    "    \n",
    "    # chexpert column\n",
    "    c_cx = f'{c}_cx'\n",
    "    \n",
    "    # annotate disagreement\n",
    "    for val in labels.keys():\n",
    "        if val == -9:\n",
    "            continue\n",
    "        \n",
    "        # check one is null and the other isn't\n",
    "        idx = df[c].isnull() & df[c_cx].notnull()\n",
    "        df.loc[idx, c] = -9\n",
    "        \n",
    "        idx = df[c].notnull() & df[c_cx].isnull()\n",
    "        df.loc[idx, c] = -9\n",
    "        \n",
    "        # check both non-null, but different value\n",
    "        idx = df[c].notnull() & df[c_cx].notnull() & (df[c] != df[c_cx])\n",
    "        df.loc[idx, c] = -9\n",
    "        \n",
    "    # now for those missing in negbio\n",
    "    idx = df[c].isnull() & df[f'{c}_cx'].notnull()\n",
    "    df.loc[idx, c] = -9\n",
    "    \n",
    "    df[c] = df[c].map(labels)\n",
    "    \n",
    "# drop chexpert columns\n",
    "cols_drop = [c for c in df.columns if c.endswith('_cx')]\n",
    "df.drop(cols_drop, axis=1, inplace=True)\n",
    "\n",
    "# display a few example cases\n",
    "display(df.head(n=10))\n",
    "\n",
    "# create a summary table of the findings\n",
    "grp_cols = [c for c in df.columns if c not in ('subject_id', 'study_id')]\n",
    "tbl = {}\n",
    "for c in grp_cols:\n",
    "    tbl[c] = df[c].value_counts().to_dict()\n",
    "tbl = pd.DataFrame.from_dict(tbl, orient='index')\n",
    "\n",
    "\n",
    "# pretty format the labels\n",
    "N = df.shape[0]\n",
    "for c in tbl.columns:\n",
    "    tbl[c] = tbl[c].apply(lambda x: f'{x:,} ({100.0*x/N:3.1f}%)')\n",
    "\n",
    "# sort columns\n",
    "print(f'Frequency of labels in MIMIC-CXR-JPG on the training subset of {df.shape[0]:,} unique radiologic studies.')\n",
    "tbl = tbl[['Positive', 'Negative', 'Uncertain', 'Disagreement']]\n",
    "tbl.to_latex('findings_frequency.tex')\n",
    "tbl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Report sectioning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['study', 'impression', 'findings', 'last_paragraph', 'comparison'], dtype='object')\n",
      "Of the total 227,835 reports.. \n",
      "  189,561 (83.2%) had a impression section\n",
      "  27,684 (12.2%) had a findings section\n",
      "  10,514 (4.6%) had a last_paragraph section\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(mimic_cxr_path / 'cxr-record-list.csv.gz', header=0, sep=',')\n",
    "sections = pd.read_csv(mimic_cxr_path / 'mimic-cxr-sections/mimic_cxr_sectioned.csv')\n",
    "\n",
    "print(sections.columns)\n",
    "N = df['study_id'].nunique()\n",
    "print(f'Of the total {N:,} reports.. ')\n",
    "idx = sections['study'].notnull()\n",
    "for c in ['impression', 'findings', 'last_paragraph']:\n",
    "    n = sections.loc[idx, c].count()\n",
    "    print(f'  {n:,} ({100.0*n/N:3.1f}%) had a {c} section')\n",
    "    # limit next check to only rows where this section is null\n",
    "    idx = idx & sections[c].isnull()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ed37",
   "language": "python",
   "name": "ed37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}