{ "cells": [ { "cell_type": "markdown", "id": "699a83ce", "metadata": { "papermill": { "duration": 0.002734, "end_time": "2024-02-19T14:35:12.487106", "exception": false, "start_time": "2024-02-19T14:35:12.484372", "status": "completed" }, "tags": [] }, "source": [ "# Feature Extraction of Base audio files from Invenio" ] }, { "cell_type": "code", "execution_count": null, "id": "6463a609", "metadata": { "collapsed": true, "execution": { "iopub.execute_input": "2024-02-19T14:35:12.495437Z", "iopub.status.busy": "2024-02-19T14:35:12.494602Z", "iopub.status.idle": "2024-02-19T14:35:13.435750Z", "shell.execute_reply": "2024-02-19T14:35:13.435185Z" }, "papermill": { "duration": 0.948005, "end_time": "2024-02-19T14:35:13.437731", "exception": false, "start_time": "2024-02-19T14:35:12.489726", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "from contextlib import contextmanager, redirect_stderr, redirect_stdout\n", "import pandas as pd\n", "import librosa\n", "import tarfile\n", "from pathlib import Path\n", "from concurrent.futures import ThreadPoolExecutor\n", "from definitions import BASE_PATH\n", "import os" ] }, { "cell_type": "code", "execution_count": null, "id": "f025335b", "metadata": { "execution": { "iopub.execute_input": "2024-02-19T14:35:13.450562Z", "iopub.status.busy": "2024-02-19T14:35:13.450137Z", "iopub.status.idle": "2024-02-19T14:35:13.455421Z", "shell.execute_reply": "2024-02-19T14:35:13.454728Z" }, "papermill": { "duration": 0.013079, "end_time": "2024-02-19T14:35:13.456656", "exception": false, "start_time": "2024-02-19T14:35:13.443577", "status": "completed" }, "tags": [ "parameters" ] }, "outputs": [], "source": [ "INPUT_PATH = BASE_PATH / \"tmp\" / \"2_generate_features\" / \"input\"\n", "OUTPUT_PATH = BASE_PATH / \"tmp\" / \"2_generate_features\" / \"output\"\n", "\n", "INPUT_PATHS: dict[str, str] = {\n", " \"audio_tar\": (INPUT_PATH / \"emotifymusic.tar.gz\").__str__()\n", "}\n", "OUTPUT_PATHS: dict[str, str] = {\n", " \"raw_features\": (OUTPUT_PATH / \"raw_features.csv\").__str__()\n", "}" ] }, { "cell_type": "code", "execution_count": null, "id": "f640e1a8", "metadata": { "execution": { "iopub.execute_input": "2024-02-19T14:35:13.463709Z", "iopub.status.busy": "2024-02-19T14:35:13.463411Z", "iopub.status.idle": "2024-02-19T14:35:13.467081Z", "shell.execute_reply": "2024-02-19T14:35:13.466383Z" }, "papermill": { "duration": 0.00948, "end_time": "2024-02-19T14:35:13.468246", "exception": false, "start_time": "2024-02-19T14:35:13.458766", "status": "completed" }, "tags": [ "injected-parameters" ] }, "outputs": [], "source": [ "# Parameters\n", "INPUT_PATHS = {\n", " \"audio_tar\": \"/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/tmp/2_generate_features/input/emotifymusic.tar.gz\"\n", "}\n", "OUTPUT_PATHS = {\n", " \"raw_features\": \"/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/tmp/2_generate_features/output/raw_features.csv\"\n", "}\n" ] }, { "cell_type": "code", "execution_count": null, "id": "10f1b3cd", "metadata": { "execution": { "iopub.execute_input": "2024-02-19T14:35:13.476383Z", "iopub.status.busy": "2024-02-19T14:35:13.476130Z", "iopub.status.idle": "2024-02-19T14:35:15.995681Z", "shell.execute_reply": "2024-02-19T14:35:15.994777Z" }, "papermill": { "duration": 2.524776, "end_time": "2024-02-19T14:35:15.997391", "exception": false, "start_time": "2024-02-19T14:35:13.472615", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# inputs\n", "\n", "DEFAULT_SAMPLING_RATE = 22050\n", "\n", "assert INPUT_PATH.exists() and INPUT_PATH.is_dir()\n", "\n", "with tarfile.open(audio_gz := Path(INPUT_PATHS[\"audio_tar\"]).resolve(), \"r:gz\") as archive:\n", " archive.extractall(path=(path_out := audio_gz.with_suffix(\"\").with_suffix(\"\")))\n", "\n", "files = list(path_out.rglob(\"**/*.*\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "469af6f9", "metadata": { "execution": { "iopub.execute_input": "2024-02-19T14:35:16.009955Z", "iopub.status.busy": "2024-02-19T14:35:16.009249Z", "iopub.status.idle": "2024-02-19T14:35:16.014300Z", "shell.execute_reply": "2024-02-19T14:35:16.013355Z" }, "papermill": { "duration": 0.015977, "end_time": "2024-02-19T14:35:16.016309", "exception": false, "start_time": "2024-02-19T14:35:16.000332", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "@contextmanager\n", "def suppress_stdout_stderr():\n", " \"\"\"A context manager that redirects stdout and stderr to devnull\"\"\"\n", " with open(os.devnull, 'w') as fnull:\n", " with redirect_stderr(fnull) as err, redirect_stdout(fnull) as out:\n", " yield err, out" ] }, { "cell_type": "code", "execution_count": null, "id": "316f6c17", "metadata": { "execution": { "iopub.execute_input": "2024-02-19T14:35:16.023046Z", "iopub.status.busy": "2024-02-19T14:35:16.022656Z", "iopub.status.idle": "2024-02-19T14:37:44.291512Z", "shell.execute_reply": "2024-02-19T14:37:44.290809Z" }, "papermill": { "duration": 148.274774, "end_time": "2024-02-19T14:37:44.293716", "exception": false, "start_time": "2024-02-19T14:35:16.018942", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def generate_mfcc_feature(filepath: Path, sr: int = DEFAULT_SAMPLING_RATE, number_mfccs: int = 40):\n", " x, _ = load_mp3(filepath, sr=sr)\n", " assert sr == _\n", " mfcc = librosa.feature.mfcc(x, sr=sr, n_mfcc=number_mfccs)\n", "\n", " # transpose to use mfcc bands as columns instead of rows\n", " return pd.DataFrame(mfcc).transpose()\n", "\n", "def load_mp3(filepath: Path, sr: int = DEFAULT_SAMPLING_RATE):\n", " x, sr = librosa.load(filepath, sr=sr) # extract wave (x) with sample rate (sr)\n", " return x, sr\n", "\n", "with suppress_stdout_stderr(), ThreadPoolExecutor(6) as executor:\n", " dataframes = list(executor.map(\n", " lambda args: generate_mfcc_feature(args), files)\n", " )" ] }, { "cell_type": "code", "execution_count": null, "id": "acc9bae8", "metadata": { "execution": { "iopub.execute_input": "2024-02-19T14:37:44.309622Z", "iopub.status.busy": "2024-02-19T14:37:44.309323Z", "iopub.status.idle": "2024-02-19T14:37:44.982496Z", "shell.execute_reply": "2024-02-19T14:37:44.981648Z" }, "papermill": { "duration": 0.683008, "end_time": "2024-02-19T14:37:44.983803", "exception": false, "start_time": "2024-02-19T14:37:44.300795", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "for file, dataframe in zip(files, dataframes):\n", " dataframe[\"sample\"] = dataframe.index.to_numpy(copy=True)\n", " dataframe[\"filename\"] = file.name\n", " dataframe[\"label\"] = file.name.split('_')[0] # extract genre from file name\n", "\n", "dataframe_concat = pd.concat(dataframes)\n", "columns_old = list(dataframe_concat.columns)\n", "columns = columns_old[-3:] + columns_old[:-3]\n", "dataframe_concat = dataframe_concat[columns]\n", "\n", "output: pd.DataFrame = dataframe_concat\n", "output" ] }, { "cell_type": "code", "execution_count": null, "id": "0abf745b", "metadata": { "execution": { "iopub.execute_input": "2024-02-19T14:37:44.992409Z", "iopub.status.busy": "2024-02-19T14:37:44.991617Z", "iopub.status.idle": "2024-02-19T14:38:17.017952Z", "shell.execute_reply": "2024-02-19T14:38:17.017278Z" }, "papermill": { "duration": 32.032086, "end_time": "2024-02-19T14:38:17.019559", "exception": false, "start_time": "2024-02-19T14:37:44.987473", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# outputs\n", "OUTPUT_PATH.mkdir(parents=True, exist_ok=True)\n", "output.to_csv(OUTPUT_PATHS[\"raw_features\"], index=False)" ] } ], "metadata": { "celltoolbar": "Tags", "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" }, "papermill": { "default_parameters": {}, "duration": 186.073807, "end_time": "2024-02-19T14:38:17.641976", "environment_variables": {}, "exception": null, "input_path": "/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/notebooks/2_generate_features.ipynb", "output_path": "/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/notebooks/2_generate_features.ipynb", "parameters": { "INPUT_PATHS": { "audio_tar": "/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/tmp/2_generate_features/input/emotifymusic.tar.gz" }, "OUTPUT_PATHS": { "raw_features": "/home/lukas/Programming/uni/bachelorarbeit/dbrepo-ismir/tmp/2_generate_features/output/raw_features.csv" } }, "start_time": "2024-02-19T14:35:11.568169", "version": "2.4.0" } }, "nbformat": 4, "nbformat_minor": 5 }