2_generate_features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "699a83ce",
   "metadata": {
    "papermill": {
     "duration": 0.002734,
     "end_time": "2024-02-19T14:35:12.487106",
     "exception": false,
     "start_time": "2024-02-19T14:35:12.484372",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Feature Extraction of Base audio files from Invenio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6463a609",
   "metadata": {
    "collapsed": true,
    "execution": {
     "iopub.execute_input": "2024-02-19T14:35:12.495437Z",
     "iopub.status.busy": "2024-02-19T14:35:12.494602Z",
     "iopub.status.idle": "2024-02-19T14:35:13.435750Z",
     "shell.execute_reply": "2024-02-19T14:35:13.435185Z"
    },
    "papermill": {
     "duration": 0.948005,
     "end_time": "2024-02-19T14:35:13.437731",
     "exception": false,
     "start_time": "2024-02-19T14:35:12.489726",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from contextlib import contextmanager, redirect_stderr, redirect_stdout\n",
    "import pandas as pd\n",
    "import librosa\n",
    "import tarfile\n",
    "from pathlib import Path\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from definitions import BASE_PATH\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f025335b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-19T14:35:13.450562Z",
     "iopub.status.busy": "2024-02-19T14:35:13.450137Z",
     "iopub.status.idle": "2024-02-19T14:35:13.455421Z",
     "shell.execute_reply": "2024-02-19T14:35:13.454728Z"
    },
    "papermill": {
     "duration": 0.013079,
     "end_time": "2024-02-19T14:35:13.456656",
     "exception": false,
     "start_time": "2024-02-19T14:35:13.443577",
     "status": "completed"
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "INPUT_PATH = BASE_PATH / \"tmp\" / \"2_generate_features\" / \"input\"\n",
    "OUTPUT_PATH = BASE_PATH / \"tmp\" / \"2_generate_features\" / \"output\"\n",
    "\n",
    "INPUT_PATHS: dict[str, str] = {\n",
    "    \"audio_tar\": (INPUT_PATH / \"emotifymusic.tar.gz\").__str__()\n",
    "}\n",
    "OUTPUT_PATHS: dict[str, str] = {\n",
    "    \"raw_features\": (OUTPUT_PATH / \"raw_features.csv\").__str__()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f640e1a8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-19T14:35:13.463709Z",
     "iopub.status.busy": "2024-02-19T14:35:13.463411Z",
     "iopub.status.idle": "2024-02-19T14:35:13.467081Z",
     "shell.execute_reply": "2024-02-19T14:35:13.466383Z"
    },
    "papermill": {
     "duration": 0.00948,
     "end_time": "2024-02-19T14:35:13.468246",
     "exception": false,
     "start_time": "2024-02-19T14:35:13.458766",
     "status": "completed"
    },
    "tags": [
     "injected-parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "INPUT_PATHS = {\n",
    "    \"audio_tar\": \"/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/tmp/2_generate_features/input/emotifymusic.tar.gz\"\n",
    "}\n",
    "OUTPUT_PATHS = {\n",
    "    \"raw_features\": \"/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/tmp/2_generate_features/output/raw_features.csv\"\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10f1b3cd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-19T14:35:13.476383Z",
     "iopub.status.busy": "2024-02-19T14:35:13.476130Z",
     "iopub.status.idle": "2024-02-19T14:35:15.995681Z",
     "shell.execute_reply": "2024-02-19T14:35:15.994777Z"
    },
    "papermill": {
     "duration": 2.524776,
     "end_time": "2024-02-19T14:35:15.997391",
     "exception": false,
     "start_time": "2024-02-19T14:35:13.472615",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# inputs\n",
    "\n",
    "DEFAULT_SAMPLING_RATE = 22050\n",
    "\n",
    "assert INPUT_PATH.exists() and INPUT_PATH.is_dir()\n",
    "\n",
    "with tarfile.open(audio_gz := Path(INPUT_PATHS[\"audio_tar\"]).resolve(), \"r:gz\") as archive:\n",
    "    archive.extractall(path=(path_out := audio_gz.with_suffix(\"\").with_suffix(\"\")))\n",
    "\n",
    "files = list(path_out.rglob(\"**/*.*\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "469af6f9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-19T14:35:16.009955Z",
     "iopub.status.busy": "2024-02-19T14:35:16.009249Z",
     "iopub.status.idle": "2024-02-19T14:35:16.014300Z",
     "shell.execute_reply": "2024-02-19T14:35:16.013355Z"
    },
    "papermill": {
     "duration": 0.015977,
     "end_time": "2024-02-19T14:35:16.016309",
     "exception": false,
     "start_time": "2024-02-19T14:35:16.000332",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "@contextmanager\n",
    "def suppress_stdout_stderr():\n",
    "    \"\"\"A context manager that redirects stdout and stderr to devnull\"\"\"\n",
    "    with open(os.devnull, 'w') as fnull:\n",
    "        with redirect_stderr(fnull) as err, redirect_stdout(fnull) as out:\n",
    "            yield err, out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "316f6c17",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-19T14:35:16.023046Z",
     "iopub.status.busy": "2024-02-19T14:35:16.022656Z",
     "iopub.status.idle": "2024-02-19T14:37:44.291512Z",
     "shell.execute_reply": "2024-02-19T14:37:44.290809Z"
    },
    "papermill": {
     "duration": 148.274774,
     "end_time": "2024-02-19T14:37:44.293716",
     "exception": false,
     "start_time": "2024-02-19T14:35:16.018942",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def generate_mfcc_feature(filepath: Path, sr: int = DEFAULT_SAMPLING_RATE, number_mfccs: int = 40):\n",
    "    x, _ = load_mp3(filepath, sr=sr)\n",
    "    assert sr == _\n",
    "    mfcc = librosa.feature.mfcc(x, sr=sr, n_mfcc=number_mfccs)\n",
    "\n",
    "    # transpose to use mfcc bands as columns instead of rows\n",
    "    return pd.DataFrame(mfcc).transpose()\n",
    "\n",
    "def load_mp3(filepath: Path, sr: int = DEFAULT_SAMPLING_RATE):\n",
    "    x, sr = librosa.load(filepath, sr=sr)   # extract wave (x) with sample rate (sr)\n",
    "    return x, sr\n",
    "\n",
    "with suppress_stdout_stderr(), ThreadPoolExecutor(6) as executor:\n",
    "    dataframes = list(executor.map(\n",
    "        lambda args: generate_mfcc_feature(args), files)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acc9bae8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-19T14:37:44.309622Z",
     "iopub.status.busy": "2024-02-19T14:37:44.309323Z",
     "iopub.status.idle": "2024-02-19T14:37:44.982496Z",
     "shell.execute_reply": "2024-02-19T14:37:44.981648Z"
    },
    "papermill": {
     "duration": 0.683008,
     "end_time": "2024-02-19T14:37:44.983803",
     "exception": false,
     "start_time": "2024-02-19T14:37:44.300795",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for file, dataframe in zip(files, dataframes):\n",
    "    dataframe[\"sample\"] = dataframe.index.to_numpy(copy=True)\n",
    "    dataframe[\"filename\"] = file.name\n",
    "    dataframe[\"label\"] = file.name.split('_')[0]    # extract genre from file name\n",
    "\n",
    "dataframe_concat = pd.concat(dataframes)\n",
    "columns_old = list(dataframe_concat.columns)\n",
    "columns = columns_old[-3:] + columns_old[:-3]\n",
    "dataframe_concat = dataframe_concat[columns]\n",
    "\n",
    "output: pd.DataFrame = dataframe_concat\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0abf745b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-19T14:37:44.992409Z",
     "iopub.status.busy": "2024-02-19T14:37:44.991617Z",
     "iopub.status.idle": "2024-02-19T14:38:17.017952Z",
     "shell.execute_reply": "2024-02-19T14:38:17.017278Z"
    },
    "papermill": {
     "duration": 32.032086,
     "end_time": "2024-02-19T14:38:17.019559",
     "exception": false,
     "start_time": "2024-02-19T14:37:44.987473",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# outputs\n",
    "OUTPUT_PATH.mkdir(parents=True, exist_ok=True)\n",
    "output.to_csv(OUTPUT_PATHS[\"raw_features\"], index=False)"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Tags",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 186.073807,
   "end_time": "2024-02-19T14:38:17.641976",
   "environment_variables": {},
   "exception": null,
   "input_path": "/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/notebooks/2_generate_features.ipynb",
   "output_path": "/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/notebooks/2_generate_features.ipynb",
   "parameters": {
    "INPUT_PATHS": {
     "audio_tar": "/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/tmp/2_generate_features/input/emotifymusic.tar.gz"
    },
    "OUTPUT_PATHS": {
     "raw_features": "/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/tmp/2_generate_features/output/raw_features.csv"
    }
   },
   "start_time": "2024-02-19T14:35:11.568169",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}