From 04386bba814852f40e43251a4428f9a604423499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Wed, 28 Dec 2022 23:32:05 +0100 Subject: [PATCH 1/4] Add audio input --- assets/js/hooks/audio_input.js | 347 ++++++++++++++++++ assets/js/hooks/image_input.js | 12 +- assets/js/hooks/index.js | 2 + .../live/output/audio_input_component.ex | 78 ++++ .../live/output/image_input_component.ex | 5 +- .../live/output/input_component.ex | 29 ++ 6 files changed, 464 insertions(+), 9 deletions(-) create mode 100644 assets/js/hooks/audio_input.js create mode 100644 lib/livebook_web/live/output/audio_input_component.ex diff --git a/assets/js/hooks/audio_input.js b/assets/js/hooks/audio_input.js new file mode 100644 index 00000000000..9c51e0d9023 --- /dev/null +++ b/assets/js/hooks/audio_input.js @@ -0,0 +1,347 @@ +import { getAttributeOrThrow, parseInteger } from "../lib/attribute"; +import { base64ToBuffer, bufferToBase64 } from "../lib/utils"; + +const dropClasses = ["bg-yellow-100", "border-yellow-300"]; + +/** + * A hook for client-preprocessed audio input. + * + * ## Configuration + * + * * `data-id` - a unique id + * + * * `data-phx-target` - the component to send the `"change"` event to + * + * * `data-format` - the desired audio format + * + * * `data-sampling-rate` - the audio sampling rate for + * + * * `data-endianness` - the server endianness, either `"little"` or `"big"` + * + */ +const AudioInput = { + mounted() { + this.props = this.getProps(); + + this.inputEl = this.el.querySelector(`[data-input]`); + this.audioEl = this.el.querySelector(`[data-preview]`); + this.uploadButton = this.el.querySelector(`[data-btn-upload]`); + this.recordButton = this.el.querySelector(`[data-btn-record]`); + this.stopButton = this.el.querySelector(`[data-btn-stop]`); + + this.mediaRecorder = null; + + // Render initial value + this.handleEvent(`audio_input_init:${this.props.id}`, (audioInfo) => { + this.updatePreview({ + data: this.decodeAudio(base64ToBuffer(audioInfo.data)), + numChannels: audioInfo.num_channels, + samplingRate: audioInfo.sampling_rate, + }); + }); + + // File selection + + this.uploadButton.addEventListener("click", (event) => { + this.inputEl.click(); + }); + + this.inputEl.addEventListener("change", (event) => { + const [file] = event.target.files; + file && this.loadFile(file); + }); + + // Drag and drop + + this.el.addEventListener("dragover", (event) => { + event.stopPropagation(); + event.preventDefault(); + event.dataTransfer.dropEffect = "copy"; + }); + + this.el.addEventListener("drop", (event) => { + event.stopPropagation(); + event.preventDefault(); + const [file] = event.dataTransfer.files; + file && !this.isRecording() && this.loadFile(file); + }); + + this.el.addEventListener("dragenter", (event) => { + this.el.classList.add(...dropClasses); + }); + + this.el.addEventListener("dragleave", (event) => { + if (!this.el.contains(event.relatedTarget)) { + this.el.classList.remove(...dropClasses); + } + }); + + this.el.addEventListener("drop", (event) => { + this.el.classList.remove(...dropClasses); + }); + + // Microphone capture + + this.recordButton.addEventListener("click", (event) => { + this.startRecording(); + }); + + this.stopButton.addEventListener("click", (event) => { + this.stopRecording(); + }); + }, + + updated() { + this.props = this.getProps(); + }, + + getProps() { + return { + id: getAttributeOrThrow(this.el, "data-id"), + phxTarget: getAttributeOrThrow(this.el, "data-phx-target", parseInteger), + samplingRate: getAttributeOrThrow( + this.el, + "data-sampling-rate", + parseInteger + ), + endianness: getAttributeOrThrow(this.el, "data-endianness"), + format: getAttributeOrThrow(this.el, "data-format"), + }; + }, + + startRecording() { + this.uploadButton.classList.add("hidden"); + this.recordButton.classList.add("hidden"); + this.stopButton.classList.remove("hidden"); + + const audioChunks = []; + + navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { + this.mediaRecorder = new MediaRecorder(stream); + + this.mediaRecorder.addEventListener("dataavailable", (event) => { + audioChunks.push(event.data); + }); + + this.mediaRecorder.addEventListener("stop", (event) => { + const audioBlob = new Blob(audioChunks); + + audioBlob.arrayBuffer().then((buffer) => { + this.loadEncodedAudio(buffer); + }); + }); + + this.mediaRecorder.start(); + }); + }, + + stopRecording() { + this.uploadButton.classList.remove("hidden"); + this.recordButton.classList.remove("hidden"); + this.stopButton.classList.add("hidden"); + + this.mediaRecorder.stop(); + }, + + isRecording() { + return this.mediaRecorder && this.mediaRecorder.state === "recording"; + }, + + loadFile(file) { + const reader = new FileReader(); + + reader.onload = (readerEvent) => { + this.loadEncodedAudio(readerEvent.target.result); + }; + + reader.readAsArrayBuffer(file); + }, + + loadEncodedAudio(buffer) { + const context = new AudioContext({ sampleRate: this.props.samplingRate }); + + context.decodeAudioData(buffer, (audioBuffer) => { + const audioInfo = audioBufferToAudioInfo(audioBuffer); + this.updatePreview(audioInfo); + this.pushAudio(audioInfo); + }); + }, + + updatePreview(audioInfo) { + const oldUrl = this.audioEl.src; + const blob = audioInfoToWavBlob(audioInfo); + this.audioEl.src = URL.createObjectURL(blob); + oldUrl && URL.revokeObjectURL(oldUrl); + }, + + pushAudio(audioInfo) { + this.pushEventTo(this.props.phxTarget, "change", { + value: { + data: bufferToBase64(this.encodeAudio(audioInfo)), + num_channels: audioInfo.numChannels, + sampling_rate: audioInfo.samplingRate, + }, + }); + }, + + encodeAudio(audioInfo) { + if (this.props.format === "pcm_f32") { + return this.fixEndianness32(audioInfo.data); + } else if (this.props.format === "wav") { + return encodeWavData( + audioInfo.data, + audioInfo.numChannels, + audioInfo.samplingRate + ); + } + }, + + decodeAudio(buffer) { + if (this.props.format === "pcm_f32") { + return this.fixEndianness32(buffer); + } else if (this.props.format === "wav") { + return decodeWavData(buffer); + } + }, + + fixEndianness32(buffer) { + if (getEndianness() === this.props.endianness) { + return buffer; + } + + // If the server uses different endianness, we swap bytes accordingly + for (const i = 0; i < buffer.byteLength / 4; i++) { + const b1 = buffer[i]; + const b2 = buffer[i + 1]; + const b3 = buffer[i + 2]; + const b4 = buffer[i + 3]; + buffer[i] = b4; + buffer[i + 1] = b3; + buffer[i + 2] = b2; + buffer[i + 3] = b1; + } + + return buffer; + }, +}; + +function audioBufferToAudioInfo(audioBuffer) { + const numChannels = audioBuffer.numberOfChannels; + const samplingRate = audioBuffer.sampleRate; + const length = audioBuffer.length; + + const size = 4 * numChannels * length; + const buffer = new ArrayBuffer(size); + + const pcmArray = new Float32Array(buffer); + + for (let channelIdx = 0; channelIdx < numChannels; channelIdx++) { + const channelArray = audioBuffer.getChannelData(channelIdx); + + for (let i = 0; i < channelArray.length; i++) { + pcmArray[numChannels * i + channelIdx] = channelArray[i]; + } + } + + return { data: pcmArray.buffer, numChannels, samplingRate }; +} + +function audioInfoToWavBlob({ data, numChannels, samplingRate }) { + const wavBytes = encodeWavData(data, numChannels, samplingRate); + return new Blob([wavBytes], { type: "audio/wav" }); +} + +// See http://soundfile.sapp.org/doc/WaveFormat +function encodeWavData(buffer, numChannels, samplingRate) { + const HEADER_SIZE = 44; + + const wavBuffer = new ArrayBuffer(HEADER_SIZE + buffer.byteLength); + const view = new DataView(wavBuffer); + + const numFrames = buffer.byteLength / 4; + const bytesPerSample = 4; + + const blockAlign = numChannels * bytesPerSample; + const byteRate = samplingRate * blockAlign; + const dataSize = numFrames * blockAlign; + + let offset = 0; + + function writeUint32Big(int) { + view.setUint32(offset, int, false); + offset += 4; + } + + function writeUint32(int) { + view.setUint32(offset, int, true); + offset += 4; + } + + function writeUint16(int) { + view.setUint16(offset, int, true); + offset += 2; + } + + function writeFloat32(int) { + view.setFloat32(offset, int, true); + offset += 4; + } + + writeUint32Big(0x52494646); + writeUint32(36 + dataSize); + writeUint32Big(0x57415645); + + writeUint32Big(0x666d7420); + writeUint32(16); + writeUint16(3); // 3 represents 32-bit float PCM + writeUint16(numChannels); + writeUint32(samplingRate); + writeUint32(byteRate); + writeUint16(blockAlign); + writeUint16(bytesPerSample * 8); + + writeUint32Big(0x64617461); + writeUint32(dataSize); + + const array = new Float32Array(buffer); + + for (let i = 0; i < array.length; i++) { + writeFloat32(array[i]); + } + + return wavBuffer; +} + +// We assume the exact same format as above, since we only need to +// decode data we encoded previously +function decodeWavData(buffer) { + const HEADER_SIZE = 44; + + const pcmBuffer = new ArrayBuffer(buffer.byteLength - HEADER_SIZE); + const pcmArray = new Float32Array(pcmBuffer); + + const view = new DataView(buffer); + + for (let i = 0; i < pcmArray.length; i++) { + const offset = HEADER_SIZE + i * 4; + pcmArray[i] = view.getFloat32(offset, true); + } + + return pcmBuffer; +} + +function getEndianness() { + const buffer = new ArrayBuffer(2); + const int16Array = new Uint16Array(buffer); + const int8Array = new Uint8Array(buffer); + + int16Array[0] = 1; + + if (int8Array[0] === 1) { + return "little"; + } else { + return "big"; + } +} + +export default AudioInput; diff --git a/assets/js/hooks/image_input.js b/assets/js/hooks/image_input.js index ba3623b92d9..94f84894ad9 100644 --- a/assets/js/hooks/image_input.js +++ b/assets/js/hooks/image_input.js @@ -48,7 +48,7 @@ const ImageInput = { // Render initial value this.handleEvent(`image_input_init:${this.props.id}`, (imageInfo) => { - const canvas = imageInfoToElement(imageInfo); + const canvas = imageInfoToElement(imageInfo, this.props.format); this.setPreview(canvas); }); @@ -421,15 +421,15 @@ function imageDataToRGBBuffer(imageData) { return bytes.buffer; } -function imageInfoToElement(imageInfo) { - if (imageInfo.format === "png" || imageInfo.format === "jpeg") { - const src = `data:image/${imageInfo.format};base64,${imageInfo.data}`; +function imageInfoToElement(imageInfo, format) { + if (format === "png" || format === "jpeg") { + const src = `data:image/${format};base64,${imageInfo.data}`; const img = document.createElement("img"); img.src = src; return img; } - if (imageInfo.format === "rgb") { + if (format === "rgb") { const canvas = document.createElement("canvas"); canvas.height = imageInfo.height; canvas.width = imageInfo.width; @@ -443,7 +443,7 @@ function imageInfoToElement(imageInfo) { return canvas; } - throw new Error(`Unexpected format: ${imageInfo.format}`); + throw new Error(`Unexpected format: ${format}`); } function imageDataFromRGBBuffer(buffer, width, height) { diff --git a/assets/js/hooks/index.js b/assets/js/hooks/index.js index e25e999e776..4200acd85ef 100644 --- a/assets/js/hooks/index.js +++ b/assets/js/hooks/index.js @@ -1,3 +1,4 @@ +import AudioInput from "./audio_input"; import Cell from "./cell"; import CellEditor from "./cell_editor"; import ConfirmModal from "./confirm_modal"; @@ -19,6 +20,7 @@ import UserForm from "./user_form"; import VirtualizedLines from "./virtualized_lines"; export default { + AudioInput, Cell, CellEditor, ConfirmModal, diff --git a/lib/livebook_web/live/output/audio_input_component.ex b/lib/livebook_web/live/output/audio_input_component.ex new file mode 100644 index 00000000000..6be12e2ab36 --- /dev/null +++ b/lib/livebook_web/live/output/audio_input_component.ex @@ -0,0 +1,78 @@ +defmodule LivebookWeb.Output.AudioInputComponent do + use LivebookWeb, :live_component + + @impl true + def mount(socket) do + {:ok, assign(socket, endianness: System.endianness(), initialized: false)} + end + + @impl true + def update(assigns, socket) do + {value, assigns} = Map.pop!(assigns, :value) + + socket = assign(socket, assigns) + + socket = + if socket.assigns.initialized do + socket + else + socket = + if value do + push_event(socket, "audio_input_init:#{socket.assigns.id}", %{ + data: Base.encode64(value.data), + num_channels: value.num_channels, + sampling_rate: value.sampling_rate + }) + else + socket + end + + assign(socket, initialized: true) + end + + {:ok, socket} + end + + @impl true + def render(assigns) do + ~H""" +
+ + +
+ + + +
+
+ """ + end +end diff --git a/lib/livebook_web/live/output/image_input_component.ex b/lib/livebook_web/live/output/image_input_component.ex index 60c8d425a9b..e31184d55af 100644 --- a/lib/livebook_web/live/output/image_input_component.ex +++ b/lib/livebook_web/live/output/image_input_component.ex @@ -21,14 +21,13 @@ defmodule LivebookWeb.Output.ImageInputComponent do push_event(socket, "image_input_init:#{socket.assigns.id}", %{ data: Base.encode64(value.data), height: value.height, - width: value.width, - format: value.format + width: value.width }) else socket end - assign(socket, initialize: true) + assign(socket, initialized: true) end {:ok, socket} diff --git a/lib/livebook_web/live/output/input_component.ex b/lib/livebook_web/live/output/input_component.ex index 4fc23cd9e85..8cc32cce7ae 100644 --- a/lib/livebook_web/live/output/input_component.ex +++ b/lib/livebook_web/live/output/input_component.ex @@ -40,6 +40,25 @@ defmodule LivebookWeb.Output.InputComponent do """ end + def render(%{attrs: %{type: :audio}} = assigns) do + ~H""" +
+
+ <%= @attrs.label %> +
+ + <.live_component + module={LivebookWeb.Output.AudioInputComponent} + id={"#{@id}-input"} + value={@value} + format={@attrs.format} + sampling_rate={@attrs.sampling_rate} + target={@myself} + /> +
+ """ + end + def render(assigns) do ~H"""
@@ -276,6 +295,16 @@ defmodule LivebookWeb.Output.InputComponent do }} end + defp parse(html_value, %{type: :audio} = attrs) do + {:ok, + %{ + data: Base.decode64!(html_value["data"]), + num_channels: html_value["num_channels"], + sampling_rate: html_value["sampling_rate"], + format: attrs.format + }} + end + defp report_event(socket, value) do topic = socket.assigns.attrs.ref event = %{value: value, origin: socket.assigns.client_id, type: :change} From 9378eedbfe3d76244140bcfb75edba821c21d29f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 29 Dec 2022 12:09:25 +0100 Subject: [PATCH 2/4] Add recording cancellation --- assets/js/hooks/audio_input.js | 33 ++++++++++++------- .../live/output/audio_input_component.ex | 11 +++++-- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/assets/js/hooks/audio_input.js b/assets/js/hooks/audio_input.js index 9c51e0d9023..7e6ba54950e 100644 --- a/assets/js/hooks/audio_input.js +++ b/assets/js/hooks/audio_input.js @@ -28,6 +28,7 @@ const AudioInput = { this.uploadButton = this.el.querySelector(`[data-btn-upload]`); this.recordButton = this.el.querySelector(`[data-btn-record]`); this.stopButton = this.el.querySelector(`[data-btn-stop]`); + this.cancelButton = this.el.querySelector(`[data-btn-cancel]`); this.mediaRecorder = null; @@ -89,6 +90,10 @@ const AudioInput = { this.stopButton.addEventListener("click", (event) => { this.stopRecording(); }); + + this.cancelButton.addEventListener("click", (event) => { + this.stopRecording(false); + }); }, updated() { @@ -110,35 +115,41 @@ const AudioInput = { }, startRecording() { + this.audioEl.classList.add("hidden"); this.uploadButton.classList.add("hidden"); this.recordButton.classList.add("hidden"); this.stopButton.classList.remove("hidden"); + this.cancelButton.classList.remove("hidden"); - const audioChunks = []; + this.audioChunks = []; navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { this.mediaRecorder = new MediaRecorder(stream); this.mediaRecorder.addEventListener("dataavailable", (event) => { - audioChunks.push(event.data); - }); - - this.mediaRecorder.addEventListener("stop", (event) => { - const audioBlob = new Blob(audioChunks); - - audioBlob.arrayBuffer().then((buffer) => { - this.loadEncodedAudio(buffer); - }); + this.audioChunks.push(event.data); }); this.mediaRecorder.start(); }); }, - stopRecording() { + stopRecording(load = true) { + this.audioEl.classList.remove("hidden"); this.uploadButton.classList.remove("hidden"); this.recordButton.classList.remove("hidden"); this.stopButton.classList.add("hidden"); + this.cancelButton.classList.add("hidden"); + + if (load) { + this.mediaRecorder.addEventListener("stop", (event) => { + const audioBlob = new Blob(this.audioChunks); + + audioBlob.arrayBuffer().then((buffer) => { + this.loadEncodedAudio(buffer); + }); + }); + } this.mediaRecorder.stop(); }, diff --git a/lib/livebook_web/live/output/audio_input_component.ex b/lib/livebook_web/live/output/audio_input_component.ex index 6be12e2ab36..c7c1ed733d3 100644 --- a/lib/livebook_web/live/output/audio_input_component.ex +++ b/lib/livebook_web/live/output/audio_input_component.ex @@ -38,7 +38,7 @@ defmodule LivebookWeb.Output.AudioInputComponent do ~H"""
-
+
+
""" From 62c7091da910fc24c0882167ce69121e63c4e02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Thu, 29 Dec 2022 12:15:44 +0100 Subject: [PATCH 3/4] Update recording indicator --- lib/livebook_web/live/output/audio_input_component.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/livebook_web/live/output/audio_input_component.ex b/lib/livebook_web/live/output/audio_input_component.ex index c7c1ed733d3..7bb5718525a 100644 --- a/lib/livebook_web/live/output/audio_input_component.ex +++ b/lib/livebook_web/live/output/audio_input_component.ex @@ -65,10 +65,14 @@ defmodule LivebookWeb.Output.AudioInputComponent do Record +
""" diff --git a/lib/livebook_web/live/output/image_input_component.ex b/lib/livebook_web/live/output/image_input_component.ex index e31184d55af..5bde9980b3c 100644 --- a/lib/livebook_web/live/output/image_input_component.ex +++ b/lib/livebook_web/live/output/image_input_component.ex @@ -56,13 +56,6 @@ defmodule LivebookWeb.Output.ImageInputComponent do
- <.menu id={"#{@id}-camera-select-menu"} position="bottom-left"> <:toggle> +
"""