test_units.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AudioData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from wavenet.audiodata import AudioData\n",
    "from wavenet.utils import list_files\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "x_len = 100\n",
    "filelist = list_files('./audio')\n",
    "ad = AudioData(filelist, x_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### __load_audio_from_wav"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio, dtype, sr = ad._load_audio_from_wav(filelist[0])\n",
    "\n",
    "print('File: {}, dtype: {}, sample rate: {}'.format(\n",
    "    filelist[0], dtype, sr))\n",
    "print('Span: [{}, {}]'.format(np.min(audio), np.max(audio)))\n",
    "print('Length: {} ({:.04f} seconds)'.format(len(audio), len(audio) / sr))\n",
    "print('# unique values: {}'.format(len(np.unique(audio))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _extract_segment: \n",
    "#### using `start_idx`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_len = 100\n",
    "y_len = 50\n",
    "start_idx = 500\n",
    "audio, _, _ = ad._load_audio_from_wav(filelist[2])\n",
    "x, y = ad._extract_segment(audio, x_len, y_len, start_idx=start_idx)\n",
    "\n",
    "# compare extracted to manually extracted example\n",
    "x_man = audio[start_idx:start_idx + x_len]\n",
    "y_man = audio[start_idx + x_len:start_idx + x_len + y_len]\n",
    "\n",
    "print('Lengths from method: {{x: {}, y: {}}}'.format(len(x), len(y)))\n",
    "print('Lengths from test: {{x: {}, y: {}}}'.format(len(x_man), len(y_man)))\n",
    "\n",
    "plt.subplot(211)\n",
    "plt.plot(range(x_len), x, 'b')\n",
    "plt.plot(range(x_len, x_len + y_len), y, 'r')\n",
    "plt.subplot(212)\n",
    "plt.plot(range(x_len), x_man, 'b')\n",
    "plt.plot(range(x_len, x_len + y_len), y_man, 'r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _extract segment\n",
    "#### no `start_idx`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_len = 100\n",
    "y_len = 50\n",
    "audio, _, _ = ad._load_audio_from_wav('./audio/classical.wav')\n",
    "x1, y1 = ad._extract_segment(audio, x_len, y_len)\n",
    "x2, y2 = ad._extract_segment(audio, x_len, y_len)\n",
    "\n",
    "# verify the two are different\n",
    "plt.subplot(211)\n",
    "plt.plot(range(x_len), x1, 'b')\n",
    "plt.plot(range(x_len, x_len + y_len), y1, 'r')\n",
    "plt.subplot(212)\n",
    "plt.plot(range(x_len), x2, 'b')\n",
    "plt.plot(range(x_len, x_len + y_len), y2, 'r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _to_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_len = 100\n",
    "y_len = 50\n",
    "audio, _, _ = ad._load_audio_from_wav('./audio/classical.wav')\n",
    "x, y = ad._extract_segment(audio, x_len, y_len)\n",
    "\n",
    "x_solo = ad._to_tensor(x)\n",
    "x_wy, y_wy = ad._to_tensor(x, y=y)\n",
    "\n",
    "print('Before (x): {}, Shape: {}'.format(type(x), x.shape))\n",
    "print('After (x-solo): {}, Shape: {}'.format(type(x_solo), x_solo.shape))\n",
    "print('After (x): {}, Shape: {}'.format(type(x_wy), x_wy.shape))\n",
    "print('')\n",
    "print('Before (y): {}, Shape: {}'.format(type(y), y.shape))\n",
    "print('After (y): {}, Shape: {}'.format(type(y_wy), y_wy.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _quantize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_len = 100\n",
    "y_len = 50\n",
    "audio, _, _ = ad._load_audio_from_wav('./audio/classical.wav')\n",
    "x, y = ad._extract_segment(audio, x_len, y_len)\n",
    "\n",
    "x = ad.encoder.encode(x)\n",
    "y = ad.encoder.encode(y)\n",
    "x_q = ad._quantize(x)\n",
    "y_q = ad._quantize(y, label=True)\n",
    "\n",
    "print('Before (x): span: {}, # unique: {}'.format(\n",
    "    (np.min(x), np.max(x)), len(np.unique(x))))\n",
    "print('After (x): span: {}, # unique: {}'.format(\n",
    "    (np.min(x_q), np.max(x_q)), len(np.unique(x_q))))\n",
    "print('Before (y): span: {}, # unique: {}'.format(\n",
    "    (np.min(y), np.max(y)), len(np.unique(y))))\n",
    "print('After (y): span: {}, # unique: {}'.format(\n",
    "    (np.min(y_q), np.max(y_q)), len(np.unique(y_q))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### save_wav"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_out = './tmp.wav'\n",
    "audio, dtype, sr = ad._load_audio_from_wav('./audio/classical.wav')\n",
    "print('Original: length: {}, dtype: {}, sample rate: {}'.format(\n",
    "    len(audio), dtype, sr))\n",
    "\n",
    "ad.save_wav(filename_out, audio, sr)\n",
    "\n",
    "audio_out, dtype_out, sr_out = ad._load_audio_from_wav(filename_out)\n",
    "print('Saved: length: {}, dtype: {}, sample rate: {}'.format(\n",
    "    len(audio_out), dtype_out, sr_out))\n",
    "\n",
    "plt.subplot(211)\n",
    "plt.plot(range(len(audio)), audio, 'b')\n",
    "plt.subplot(212)\n",
    "plt.plot(range(len(audio_out)), audio_out, 'b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### label2value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_len = 100\n",
    "y_len = 50\n",
    "audio, _, _ = ad._load_audio_from_wav('./audio/classical.wav')\n",
    "x, y = ad._extract_segment(audio, x_len, y_len)\n",
    "\n",
    "y_enc = ad.encoder.encode(y)\n",
    "y_q = ad._quantize(y_enc, label=True)\n",
    "\n",
    "y_val = ad.label2value(y_q)\n",
    "\n",
    "idxs = np.random.randint(y_len, size=6)\n",
    "print('Before: {}'.format(y_enc[idxs]))\n",
    "print('Quant: {}'.format(y_q[idxs]))\n",
    "print('After: {}'.format(y_val[idxs]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_len = 100\n",
    "y_len = 50\n",
    "audio, _, _ = ad._load_audio_from_wav('./audio/classical.wav')\n",
    "x, y = ad._extract_segment(audio, x_len, y_len)\n",
    "\n",
    "x_solo = ad.preprocess(x)\n",
    "x_wy, y_wy = ad.preprocess(x, y=y)\n",
    "\n",
    "print('Before (x): span: {}, # unique: {}'.format(\n",
    "    (np.min(x), np.max(x)), len(np.unique(x))))\n",
    "print('After (x-solo): span: {}, # unique: {}'.format(\n",
    "    (np.min(x_solo), np.max(x_solo)), len(np.unique(x_solo))))\n",
    "print('After (x): span: {}, # unique: {}'.format(\n",
    "    (np.min(x_wy), np.max(x_wy)), len(np.unique(x_wy))))\n",
    "print('')\n",
    "print('Before (y): span: {}, # unique: {}'.format(\n",
    "    (np.min(y), np.max(y)), len(np.unique(y))))\n",
    "print('After (y): span: {}, # unique: {}'.format(\n",
    "    (np.min(y_wy), np.max(y_wy)), len(np.unique(y_wy))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MuEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import copy\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from wavenet.audiodata import AudioData\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "x_len = 100\n",
    "filelist = ['./audio/classical.wav']\n",
    "ad = AudioData(filelist, x_len)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio, dtype, sr = ad._load_audio_from_wav(filelist[0])\n",
    "print('x: span: {}, # unique: {}'.format(\n",
    "    (np.min(audio), np.max(audio)), len(np.unique(audio))))\n",
    "\n",
    "# specify span using strings\n",
    "x_s_dr = ad.encoder.normalize(audio, span='datarange')\n",
    "x_s_mm = ad.encoder.normalize(audio, span='minmax')\n",
    "print('span=\"datarange\": span: {}, # unique: {}'.format(\n",
    "    (np.min(x_s_dr), np.max(x_s_dr)), len(np.unique(x_s_dr))))\n",
    "print('span=\"minmax\": span: {}, # unique: {}'.format(\n",
    "    (np.min(x_s_mm), np.max(x_s_mm)), len(np.unique(x_s_mm))))\n",
    "\n",
    "# specify span using span=(min, max)\n",
    "x_set = ad.encoder.normalize(audio, span=(-20000, 20000))\n",
    "print('span=span: span: {}, # unique: {}'.format(\n",
    "    (np.min(x_set), np.max(x_set)), len(np.unique(x_set))))\n",
    "\n",
    "# span=None, default to datarange\n",
    "x_dr = ad.encoder.normalize(audio)\n",
    "print('span=None: span: {}, # unique: {}'.format(\n",
    "    (np.min(x_dr), np.max(x_dr)), len(np.unique(x_dr))))\n",
    "\n",
    "# span=None and datarange=None\n",
    "ad_copy = copy.deepcopy(ad)\n",
    "ad_copy.encoder.datarange = None\n",
    "x_mm = ad_copy.encoder.normalize(audio)\n",
    "print('span=None, datarange=None: span: {}, # unique: {}'.format(\n",
    "    (np.min(x_mm), np.max(x_mm)), len(np.unique(x_mm))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### expand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, _, _ = ad._load_audio_from_wav(filelist[0])\n",
    "print('x: span: {}, # unique: {}'.format(\n",
    "    (np.min(x), np.max(x)), len(np.unique(x))))\n",
    "\n",
    "x_norm = ad.encoder.normalize(x)\n",
    "print('x normed: span: {}, # unique: {}'.format(\n",
    "    (np.min(x_norm), np.max(x_norm)), len(np.unique(x_norm))))\n",
    "\n",
    "x_expand = ad.encoder.expand(x_norm)\n",
    "print('x expanded: span: {}, # unique: {}'.format(\n",
    "    (np.min(x_expand), np.max(x_expand)), len(np.unique(x_expand))))\n",
    "\n",
    "plt.subplot(211)\n",
    "plt.plot(range(len(x)), x, 'b')\n",
    "plt.subplot(212)\n",
    "plt.plot(range(len(x_expand)), x_expand, 'b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### encode & decode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, _, _ = ad._load_audio_from_wav(filelist[0])\n",
    "print('x: span: {}, # unique: {}'.format(\n",
    "    (np.min(x), np.max(x)), len(np.unique(x))))\n",
    "\n",
    "x_enc = ad.encoder.encode(x)\n",
    "print('x encoded: span: {}, # unique: {}'.format(\n",
    "    (np.min(x_enc), np.max(x_enc)), len(np.unique(x_enc))))\n",
    "\n",
    "x_dec = ad.encoder.decode(x_enc)\n",
    "print('x decoded: span: {}, # unique: {}'.format(\n",
    "    (np.min(x_dec), np.max(x_dec)), len(np.unique(x_dec))))\n",
    "\n",
    "plt.subplot(311)\n",
    "plt.plot(range(len(x)), x, 'b')\n",
    "plt.subplot(312)\n",
    "plt.plot(range(len(x_enc)), x_enc, 'b')\n",
    "plt.subplot(313)\n",
    "plt.plot(range(len(x_dec)), x_dec, 'b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from wavenet.audiodata import AudioData, AudioLoader\n",
    "from wavenet.models import Model\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "x_len = 2**10\n",
    "num_classes = 256\n",
    "num_layers = 9\n",
    "num_blocks = 2\n",
    "num_hidden = 128\n",
    "kernel_size = 2\n",
    "learn_rate = 0.001\n",
    "step_size = 50\n",
    "gamma = 0.5\n",
    "batch_size = 8\n",
    "num_workers = 1\n",
    "num_epochs = 10\n",
    "\n",
    "filelist = ['./audio/classical.wav']\n",
    "dataset = AudioData(filelist, x_len, num_classes=num_classes, \n",
    "                    store_tracks=True)\n",
    "dataloader = AudioLoader(dataset, batch_size=batch_size, \n",
    "                         num_workers=num_workers)\n",
    "wave_model = Model(x_len, num_channels=1, num_classes=num_classes, \n",
    "                   num_blocks=num_blocks, num_layers=num_layers,\n",
    "                   num_hidden=num_hidden, kernel_size=kernel_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### set_device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Original device: {}'.format(wave_model.device))\n",
    "\n",
    "wave_model.set_device(torch.device('cpu'))\n",
    "print('Manually set device: {}'.format(wave_model.device))\n",
    "\n",
    "wave_model.set_device()\n",
    "print('Auto set device: {}'.format(wave_model.device))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wave_model.criterion = nn.CrossEntropyLoss()\n",
    "wave_model.optimizer = optim.Adam(wave_model.parameters(), \n",
    "                                  lr=learn_rate)\n",
    "wave_model.scheduler = optim.lr_scheduler.StepLR(wave_model.optimizer, \n",
    "                                                 step_size=step_size, \n",
    "                                                 gamma=gamma)\n",
    "\n",
    "wave_model.train(dataloader, num_epochs=num_epochs, disp_interval=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from wavenet.audiodata import AudioData, AudioLoader\n",
    "from wavenet.models import Model, Generator\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "x_len = 2**8\n",
    "num_classes = 256\n",
    "num_layers = 7\n",
    "num_blocks = 2\n",
    "num_hidden = 128\n",
    "kernel_size = 2\n",
    "learn_rate = 0.001\n",
    "step_size = 50\n",
    "gamma = 0.5\n",
    "batch_size = 8\n",
    "num_workers = 1\n",
    "num_epochs = 10\n",
    "model_file = 'model-7.pt'\n",
    "\n",
    "filelist = ['./audio/classical.wav']\n",
    "dataset = AudioData(filelist, x_len, num_classes=num_classes, \n",
    "                    store_tracks=True)\n",
    "dataloader = AudioLoader(dataset, batch_size=batch_size, \n",
    "                         num_workers=num_workers)\n",
    "wave_model = Model(x_len, num_channels=1, num_classes=num_classes, \n",
    "                   num_blocks=num_blocks, num_layers=num_layers,\n",
    "                   num_hidden=num_hidden, kernel_size=kernel_size)\n",
    "\n",
    "if os.path.isfile(model_file):\n",
    "    wave_model.load_state_dict(torch.load(model_file))\n",
    "else:\n",
    "    wave_model.criterion = nn.CrossEntropyLoss()\n",
    "    wave_model.optimizer = optim.Adam(wave_model.parameters(), \n",
    "                                      lr=learn_rate)\n",
    "    wave_model.scheduler = optim.lr_scheduler.StepLR(wave_model.optimizer, \n",
    "                                                     step_size=step_size, \n",
    "                                                     gamma=gamma)\n",
    "\n",
    "    wave_model.set_device(torch.device('cpu'))\n",
    "    wave_model.train(dataloader, num_epochs=num_epochs, disp_interval=1)\n",
    "    torch.save(wave_model.state_dict(), model_file)\n",
    "\n",
    "wave_generator = Generator(wave_model, dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _shift_insert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples = 3\n",
    "x = np.linspace(-2**15, 2**15, 6)\n",
    "print('x:')\n",
    "print('\\tspan: {}, # unique: {}'.format(\n",
    "    (np.min(x), np.max(x)), len(np.unique(x))))\n",
    "\n",
    "x = dataset._to_tensor(dataset.preprocess(x))\n",
    "x = torch.unsqueeze(x, 0)\n",
    "print('\\ttype: {}, shape: {}'.format(type(x), x.shape))\n",
    "print('\\tdata: {}'.format(x.data))\n",
    "\n",
    "x_s = wave_generator._shift_insert(x, np.array([0]))\n",
    "print('x shifted:')\n",
    "print('\\ttype: {}, shape: {}'.format(type(x_s), x_s.shape))\n",
    "print('\\tdata: {}'.format(x_s.data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tensor2numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, _, _ = dataset._load_audio_from_wav(filelist[0])\n",
    "x = dataset._to_tensor(dataset.preprocess(x))\n",
    "x = torch.unsqueeze(x, 0)\n",
    "print('x: type: {}, shape: {}'.format(type(x), x.shape))\n",
    "\n",
    "x_np = wave_generator.tensor2numpy(x)\n",
    "print('x numpy: type: {}, shape: {}'.format(type(x_np), x_np.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, _, _ = dataset._load_audio_from_wav(filelist[0])\n",
    "x = dataset._to_tensor(dataset.preprocess(x))\n",
    "x = torch.unsqueeze(x, 0)\n",
    "\n",
    "y = wave_generator.predict(x)\n",
    "print('y: type: {}, shape: {}'.format(type(y), y.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_samples = 100\n",
    "start_idx = 10000\n",
    "\n",
    "audio, _, _ = dataset._load_audio_from_wav(filelist[0])\n",
    "x = audio[start_idx:start_idx + x_len]\n",
    "x_samp = audio[start_idx:start_idx + x_len + num_samples]\n",
    "\n",
    "y = wave_generator.run(x, num_samples, disp_interval=10)\n",
    "print('y:')\n",
    "print('\\ntype: {}, shape: {}'.format(type(y), y.shape))\n",
    "print('\\nspan: {}, # unique: {}'.format(\n",
    "    (np.min(y), np.max(y)), len(np.unique(y))))\n",
    "\n",
    "plt.plot(range(x_len + num_samples), x_samp, 'b')\n",
    "plt.plot(range(x_len, x_len + num_samples), y, 'r')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:PyTorchSIIM]",
   "language": "python",
   "name": "conda-env-PyTorchSIIM-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}