Add support for audio input

assets/js/hooks/audio_input.js

@@ -0,0 +1,347 @@
+import { getAttributeOrThrow, parseInteger } from "../lib/attribute";
+import { base64ToBuffer, bufferToBase64 } from "../lib/utils";
+
+const dropClasses = ["bg-yellow-100", "border-yellow-300"];
+
+/**
+ * A hook for client-preprocessed audio input.
+ *
+ * ## Configuration
+ *
+ *   * `data-id` - a unique id
+ *
+ *   * `data-phx-target` - the component to send the `"change"` event to
+ *
+ *   * `data-format` - the desired audio format
+ *
+ *   * `data-sampling-rate` - the audio sampling rate for
+ *
+ *   * `data-endianness` - the server endianness, either `"little"` or `"big"`
+ *
+ */
+const AudioInput = {
+  mounted() {
+    this.props = this.getProps();
+
+    this.inputEl = this.el.querySelector(`[data-input]`);
+    this.audioEl = this.el.querySelector(`[data-preview]`);
+    this.uploadButton = this.el.querySelector(`[data-btn-upload]`);
+    this.recordButton = this.el.querySelector(`[data-btn-record]`);
+    this.stopButton = this.el.querySelector(`[data-btn-stop]`);
+
+    this.mediaRecorder = null;
+
+    // Render initial value
+    this.handleEvent(`audio_input_init:${this.props.id}`, (audioInfo) => {
+      this.updatePreview({
+        data: this.decodeAudio(base64ToBuffer(audioInfo.data)),
+        numChannels: audioInfo.num_channels,
+        samplingRate: audioInfo.sampling_rate,
+      });
+    });
+
+    // File selection
+
+    this.uploadButton.addEventListener("click", (event) => {
+      this.inputEl.click();
+    });
+
+    this.inputEl.addEventListener("change", (event) => {
+      const [file] = event.target.files;
+      file && this.loadFile(file);
+    });
+
+    // Drag and drop
+
+    this.el.addEventListener("dragover", (event) => {
+      event.stopPropagation();
+      event.preventDefault();
+      event.dataTransfer.dropEffect = "copy";
+    });
+
+    this.el.addEventListener("drop", (event) => {
+      event.stopPropagation();
+      event.preventDefault();
+      const [file] = event.dataTransfer.files;
+      file && !this.isRecording() && this.loadFile(file);
+    });
+
+    this.el.addEventListener("dragenter", (event) => {
+      this.el.classList.add(...dropClasses);
+    });
+
+    this.el.addEventListener("dragleave", (event) => {
+      if (!this.el.contains(event.relatedTarget)) {
+        this.el.classList.remove(...dropClasses);
+      }
+    });
+
+    this.el.addEventListener("drop", (event) => {
+      this.el.classList.remove(...dropClasses);
+    });
+
+    // Microphone capture
+
+    this.recordButton.addEventListener("click", (event) => {
+      this.startRecording();
+    });
+
+    this.stopButton.addEventListener("click", (event) => {
+      this.stopRecording();
+    });
+  },
+
+  updated() {
+    this.props = this.getProps();
+  },
+
+  getProps() {
+    return {
+      id: getAttributeOrThrow(this.el, "data-id"),
+      phxTarget: getAttributeOrThrow(this.el, "data-phx-target", parseInteger),
+      samplingRate: getAttributeOrThrow(
+        this.el,
+        "data-sampling-rate",
+        parseInteger
+      ),
+      endianness: getAttributeOrThrow(this.el, "data-endianness"),
+      format: getAttributeOrThrow(this.el, "data-format"),
+    };
+  },
+
+  startRecording() {
+    this.uploadButton.classList.add("hidden");
+    this.recordButton.classList.add("hidden");
+    this.stopButton.classList.remove("hidden");
+
+    const audioChunks = [];
+
+    navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
+      this.mediaRecorder = new MediaRecorder(stream);
+
+      this.mediaRecorder.addEventListener("dataavailable", (event) => {
+        audioChunks.push(event.data);
+      });
+
+      this.mediaRecorder.addEventListener("stop", (event) => {
+        const audioBlob = new Blob(audioChunks);
+
+        audioBlob.arrayBuffer().then((buffer) => {
+          this.loadEncodedAudio(buffer);
+        });
+      });
+
+      this.mediaRecorder.start();
+    });
+  },
+
+  stopRecording() {
+    this.uploadButton.classList.remove("hidden");
+    this.recordButton.classList.remove("hidden");
+    this.stopButton.classList.add("hidden");
+
+    this.mediaRecorder.stop();
+  },
+
+  isRecording() {
+    return this.mediaRecorder && this.mediaRecorder.state === "recording";
+  },
+
+  loadFile(file) {
+    const reader = new FileReader();
+
+    reader.onload = (readerEvent) => {
+      this.loadEncodedAudio(readerEvent.target.result);
+    };
+
+    reader.readAsArrayBuffer(file);
+  },
+
+  loadEncodedAudio(buffer) {
+    const context = new AudioContext({ sampleRate: this.props.samplingRate });
+
+    context.decodeAudioData(buffer, (audioBuffer) => {
+      const audioInfo = audioBufferToAudioInfo(audioBuffer);
+      this.updatePreview(audioInfo);
+      this.pushAudio(audioInfo);
+    });
+  },
+
+  updatePreview(audioInfo) {
+    const oldUrl = this.audioEl.src;
+    const blob = audioInfoToWavBlob(audioInfo);
+    this.audioEl.src = URL.createObjectURL(blob);
+    oldUrl && URL.revokeObjectURL(oldUrl);
+  },
+
+  pushAudio(audioInfo) {
+    this.pushEventTo(this.props.phxTarget, "change", {
+      value: {
+        data: bufferToBase64(this.encodeAudio(audioInfo)),
+        num_channels: audioInfo.numChannels,
+        sampling_rate: audioInfo.samplingRate,
+      },
+    });
+  },
+
+  encodeAudio(audioInfo) {
+    if (this.props.format === "pcm_f32") {
+      return this.fixEndianness32(audioInfo.data);
+    } else if (this.props.format === "wav") {
+      return encodeWavData(
+        audioInfo.data,
+        audioInfo.numChannels,
+        audioInfo.samplingRate
+      );
+    }
+  },
+
+  decodeAudio(buffer) {
+    if (this.props.format === "pcm_f32") {
+      return this.fixEndianness32(buffer);
+    } else if (this.props.format === "wav") {
+      return decodeWavData(buffer);
+    }
+  },
+
+  fixEndianness32(buffer) {
+    if (getEndianness() === this.props.endianness) {
+      return buffer;
+    }
+
+    // If the server uses different endianness, we swap bytes accordingly
+    for (const i = 0; i < buffer.byteLength / 4; i++) {
+      const b1 = buffer[i];
+      const b2 = buffer[i + 1];
+      const b3 = buffer[i + 2];
+      const b4 = buffer[i + 3];
+      buffer[i] = b4;
+      buffer[i + 1] = b3;
+      buffer[i + 2] = b2;
+      buffer[i + 3] = b1;
+    }
+
+    return buffer;
+  },
+};
+
+function audioBufferToAudioInfo(audioBuffer) {
+  const numChannels = audioBuffer.numberOfChannels;
+  const samplingRate = audioBuffer.sampleRate;
+  const length = audioBuffer.length;
+
+  const size = 4 * numChannels * length;
+  const buffer = new ArrayBuffer(size);
+
+  const pcmArray = new Float32Array(buffer);
+
+  for (let channelIdx = 0; channelIdx < numChannels; channelIdx++) {
+    const channelArray = audioBuffer.getChannelData(channelIdx);
+
+    for (let i = 0; i < channelArray.length; i++) {
+      pcmArray[numChannels * i + channelIdx] = channelArray[i];
+    }
+  }
+
+  return { data: pcmArray.buffer, numChannels, samplingRate };
+}
+
+function audioInfoToWavBlob({ data, numChannels, samplingRate }) {
+  const wavBytes = encodeWavData(data, numChannels, samplingRate);
+  return new Blob([wavBytes], { type: "audio/wav" });
+}
+
+// See http://soundfile.sapp.org/doc/WaveFormat
+function encodeWavData(buffer, numChannels, samplingRate) {
+  const HEADER_SIZE = 44;
+
+  const wavBuffer = new ArrayBuffer(HEADER_SIZE + buffer.byteLength);
+  const view = new DataView(wavBuffer);
+
+  const numFrames = buffer.byteLength / 4;
+  const bytesPerSample = 4;
+
+  const blockAlign = numChannels * bytesPerSample;
+  const byteRate = samplingRate * blockAlign;
+  const dataSize = numFrames * blockAlign;
+
+  let offset = 0;
+
+  function writeUint32Big(int) {
+    view.setUint32(offset, int, false);
+    offset += 4;
+  }
+
+  function writeUint32(int) {
+    view.setUint32(offset, int, true);
+    offset += 4;
+  }
+
+  function writeUint16(int) {
+    view.setUint16(offset, int, true);
+    offset += 2;
+  }
+
+  function writeFloat32(int) {
+    view.setFloat32(offset, int, true);
+    offset += 4;
+  }
+
+  writeUint32Big(0x52494646);
+  writeUint32(36 + dataSize);
+  writeUint32Big(0x57415645);
+
+  writeUint32Big(0x666d7420);
+  writeUint32(16);
+  writeUint16(3); // 3 represents 32-bit float PCM
+  writeUint16(numChannels);
+  writeUint32(samplingRate);
+  writeUint32(byteRate);
+  writeUint16(blockAlign);
+  writeUint16(bytesPerSample * 8);
+
+  writeUint32Big(0x64617461);
+  writeUint32(dataSize);
+
+  const array = new Float32Array(buffer);
+
+  for (let i = 0; i < array.length; i++) {
+    writeFloat32(array[i]);
+  }
+
+  return wavBuffer;
+}
+
+// We assume the exact same format as above, since we only need to
+// decode data we encoded previously
+function decodeWavData(buffer) {
+  const HEADER_SIZE = 44;
+
+  const pcmBuffer = new ArrayBuffer(buffer.byteLength - HEADER_SIZE);
+  const pcmArray = new Float32Array(pcmBuffer);
+
+  const view = new DataView(buffer);
+
+  for (let i = 0; i < pcmArray.length; i++) {
+    const offset = HEADER_SIZE + i * 4;
+    pcmArray[i] = view.getFloat32(offset, true);
+  }
+
+  return pcmBuffer;
+}
+
+function getEndianness() {
+  const buffer = new ArrayBuffer(2);
+  const int16Array = new Uint16Array(buffer);
+  const int8Array = new Uint8Array(buffer);
+
+  int16Array[0] = 1;
+
+  if (int8Array[0] === 1) {
+    return "little";
+  } else {
+    return "big";
+  }
+}
+
+export default AudioInput;

assets/js/hooks/image_input.js

@@ -48,7 +48,7 @@ const ImageInput = {
 
     // Render initial value
     this.handleEvent(`image_input_init:${this.props.id}`, (imageInfo) => {
-      const canvas = imageInfoToElement(imageInfo);
+      const canvas = imageInfoToElement(imageInfo, this.props.format);
       this.setPreview(canvas);
     });
 
@@ -421,15 +421,15 @@ function imageDataToRGBBuffer(imageData) {
   return bytes.buffer;
 }
 
-function imageInfoToElement(imageInfo) {
-  if (imageInfo.format === "png" || imageInfo.format === "jpeg") {
-    const src = `data:image/${imageInfo.format};base64,${imageInfo.data}`;
+function imageInfoToElement(imageInfo, format) {
+  if (format === "png" || format === "jpeg") {
+    const src = `data:image/${format};base64,${imageInfo.data}`;
     const img = document.createElement("img");
     img.src = src;
     return img;
   }
 
-  if (imageInfo.format === "rgb") {
+  if (format === "rgb") {
     const canvas = document.createElement("canvas");
     canvas.height = imageInfo.height;
     canvas.width = imageInfo.width;
@@ -443,7 +443,7 @@ function imageInfoToElement(imageInfo) {
     return canvas;
   }
 
-  throw new Error(`Unexpected format: ${imageInfo.format}`);
+  throw new Error(`Unexpected format: ${format}`);
 }
 
 function imageDataFromRGBBuffer(buffer, width, height) {

assets/js/hooks/index.js

@@ -1,3 +1,4 @@
+import AudioInput from "./audio_input";
 import Cell from "./cell";
 import CellEditor from "./cell_editor";
 import ConfirmModal from "./confirm_modal";
@@ -19,6 +20,7 @@ import UserForm from "./user_form";
 import VirtualizedLines from "./virtualized_lines";
 
 export default {
+  AudioInput,
   Cell,
   CellEditor,
   ConfirmModal,