audio_reader_for_scikit_talk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "z061kHF6-FLA"
   },
   "source": [
    "# hparams only stores some variables, can be removed as a dependency and variables defined directly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set up dependencies:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import Parallel\n",
    "import librosa\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "muBexQBolHxC"
   },
   "outputs": [],
   "source": [
    "class HParams(object):\n",
    "    \"\"\" Hparams was removed from tf 2.0alpha so this is a placeholder\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, **kwargs):\n",
    "        self.set_defaults()\n",
    "        self.__dict__.update(kwargs)\n",
    "\n",
    "    def set_defaults(self):\n",
    "        self.win_length_ms = 5\n",
    "        self.hop_length_ms = 1\n",
    "        self.n_fft = 1024\n",
    "        self.ref_level_db = 20\n",
    "        self.min_level_db = -60\n",
    "        self.preemphasis = 0.97\n",
    "        self.num_mel_bins = 64\n",
    "        self.mel_lower_edge_hertz = 200\n",
    "        self.mel_upper_edge_hertz = 15000\n",
    "        self.power = 1.5  # for spectral inversion\n",
    "        self.griffin_lim_iters = 50\n",
    "        self.butter_lowcut = 500\n",
    "        self.butter_highcut = 15000\n",
    "        self.reduce_noise = False\n",
    "        self.noise_reduce_kwargs = {}\n",
    "        self.mask_spec = False\n",
    "        self.mask_spec_kwargs = {\"spec_thresh\": 0.9, \"offset\": 1e-10}\n",
    "        self.nex = -1\n",
    "        self.n_jobs = -1\n",
    "        self.verbosity = 1\n",
    "\n",
    "    def save(self):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def load(self):\n",
    "        raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "vUibRcpsZuFb"
   },
   "outputs": [],
   "source": [
    "# create a set of hyperparameters for processing a dataset.\n",
    "hparams = HParams(\n",
    "    num_mel_bins = 64,\n",
    "    mel_lower_edge_hertz=500,\n",
    "    mel_upper_edge_hertz=15000,\n",
    "    butter_lowcut = 500,\n",
    "    butter_highcut = 15000,\n",
    "    ref_level_db = 20,\n",
    "    min_level_db = -30,\n",
    "    mask_spec = True,\n",
    "    win_length_ms = 10,\n",
    "    hop_length_ms = 2,\n",
    "    nex=-1,\n",
    "    n_jobs=-1,\n",
    "    verbosity = 1,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rtZclmcW-j3a"
   },
   "source": [
    "# The four helper functions  `int16_to_float32` `read_wav` `load_wav` `prepare_wav' can also be removed/merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "oeTApNFSkiS8"
   },
   "outputs": [],
   "source": [
    "def read_wav(wav_loc, method=\"librosa\", **kwargs):\n",
    "    \"\"\" read wav using either librosa or scipy\n",
    "    \"\"\"\n",
    "    if method == \"librosa\":\n",
    "        if \"sr\" not in kwargs.keys():\n",
    "            kwargs[\"sr\"] = None\n",
    "        data, rate = librosa.core.load(wav_loc, **kwargs)\n",
    "    elif method == \"scipy\":\n",
    "        rate, data = wavfile.read(wav_loc)\n",
    "    return rate, data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "4T3FNdbdkkEf"
   },
   "outputs": [],
   "source": [
    "def load_wav(wav_loc, catch_errors=True, method=\"librosa\", **kwargs):\n",
    "    if catch_errors:\n",
    "        try:\n",
    "            rate, data = read_wav(wav_loc, method=method, **kwargs)\n",
    "            return rate, data\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            return None, None\n",
    "    else:\n",
    "        rate, data = read_wav(wav_loc, method=method, **kwargs)\n",
    "        return rate, data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "85uYuVM9kZx3"
   },
   "outputs": [],
   "source": [
    "# def int16_to_float32(data):\n",
    "#     \"\"\" Converts from uint16 wav to float32 wav\n",
    "#     \"\"\"\n",
    "#     if np.max(np.abs(data)) > 32768:\n",
    "#         raise ValueError(\"Data has values above 32768\")\n",
    "#     return (data / 32768.0).astype(\"float32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "XoVqkQoHXyzd"
   },
   "outputs": [],
   "source": [
    "def prepare_wav(wav_loc, hparams=None):\n",
    "    \"\"\" load wav and convert to correct format\n",
    "    \"\"\"\n",
    "\n",
    "    # get rate and date\n",
    "    rate, data = load_wav(wav_loc)\n",
    "\n",
    "    # #butter_bandpass and reduce noise not necessary for our dataset\n",
    "    # if np.issubdtype(type(data[0]), np.integer):\n",
    "    #     data = int16_to_float32(data)\n",
    "    # # bandpass filter\n",
    "    # if hparams is not None:\n",
    "    #     data = butter_bandpass_filter(\n",
    "    #         data, hparams.butter_lowcut, hparams.butter_highcut, rate, order=5\n",
    "    #     )\n",
    "\n",
    "    #     # reduce noise\n",
    "    #     if hparams.reduce_noise:\n",
    "    #         data = nr.reduce_noise(\n",
    "    #             audio_clip=data, noise_clip=data, **hparams.noise_reduce_kwargs\n",
    "    #         )\n",
    "\n",
    "    return rate, data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LGnTL7Yg9wk2"
   },
   "source": [
    "# This is the core function to read audio."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "fZZuTJQfXSN6"
   },
   "outputs": [],
   "source": [
    "def get_row_audio(test_input, wav_loc, hparams):\n",
    "    \"\"\" load audio and grab individual snippets\n",
    "    TODO: for large sparse WAV files, the audio should be loaded only for the syllable\n",
    "    \"\"\"\n",
    "\n",
    "    # load audio\n",
    "    rate, data = prepare_wav(wav_loc, hparams)\n",
    "    data = data.astype('float32')\n",
    "\n",
    "    # get audio for each word (row) in dataframe\n",
    "    test_input[\"audio\"] = [\n",
    "        data[int(st * rate) : int(et * rate)]\n",
    "        for st, et in zip(test_input.start_time.values, test_input.end_time.values)\n",
    "    ]\n",
    "\n",
    "    test_input[\"rate\"] = rate\n",
    "\n",
    "    return test_input\n",
    "\n",
    "# Directly use\n",
    "# input = pandas.readcsv(\"testinput.csv\") # Create a dummy dataframe\n",
    "# get_row_audio(input, \"path/to/audio.wav\", hparams)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply it to a single row:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_df = pd.read_csv(\"test_input.csv\")\n",
    "row_df = input_df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_66861/2335671276.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  test_input[\"audio\"] = [\n",
      "/tmp/ipykernel_66861/2335671276.py:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  test_input[\"rate\"] = rate\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>participant</th>\n",
       "      <th>utterance</th>\n",
       "      <th>key</th>\n",
       "      <th>language</th>\n",
       "      <th>uid</th>\n",
       "      <th>audio</th>\n",
       "      <th>rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>629.96</td>\n",
       "      <td>630.51</td>\n",
       "      <td>A</td>\n",
       "      <td>aha</td>\n",
       "      <td>/german1/5298</td>\n",
       "      <td>german</td>\n",
       "      <td>german-059-255-629960</td>\n",
       "      <td>[-0.00012207031, -0.00061035156, -0.0008544922...</td>\n",
       "      <td>8000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   start_time  end_time participant utterance            key language  \\\n",
       "0      629.96    630.51           A       aha  /german1/5298   german   \n",
       "\n",
       "                     uid                                              audio  \\\n",
       "0  german-059-255-629960  [-0.00012207031, -0.00061035156, -0.0008544922...   \n",
       "\n",
       "   rate  \n",
       "0  8000  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = \"audio_files\"\n",
    "wav_loc = path + row_df.key[0] + \".wav\" # Consider changing wav_loc for path as input\n",
    "get_row_audio(row_df, wav_loc, hparams)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4drw5a2-AAte"
   },
   "source": [
    "# This currently loops over separate dataframes \"test_inputs\" but should just get audio from one dataframe test_input..\n",
    "\n",
    "Note the loop below is designed to loop among multiple data frames."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AgVpsjEOZMm2"
   },
   "outputs": [],
   "source": [
    "with Parallel(n_jobs=n_jobs, verbose=verbosity) as parallel:\n",
    "    test_inputs = parallel(\n",
    "        delayed(get_row_audio)(\n",
    "            test_input[test_input.key == key],\n",
    "            # Edit path to audio folder here\n",
    "            #'/Volumes/WRKGRP/FDL-CLS-MDingemanse-ConverseWG/Elpaco dataset'+ key +'.wav',\n",
    "            '../../testing/Elpaco dataset'+ key +'.wav', # <<<<<< Change the path\n",
    "            #'/testing/Elpaco dataset'+ key +'.wav',\n",
    "            dataset.hparams\n",
    "        )\n",
    "        for key in tqdm(test_input.key.unique())\n",
    "    )\n",
    "test_output = pd.concat(test_inputs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tyisk1EXAfvV"
   },
   "source": [
    "# After extracting the audio snippets we normalize the audio column using librosa. this needs to be integrated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WlHpmSaBiXyF"
   },
   "outputs": [],
   "source": [
    "test_output['audio'] = [librosa.util.normalize(i) for i in test_output.audio.values]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bnOuHtuGAv1a"
   },
   "source": [
    "# optional: write out file as \"test_output.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZUf0nu1IAu_3"
   },
   "outputs": [],
   "source": [
    "test_output.to_csv(\"test_output.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "ffmpeg-test",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}