Add custom loss functions and a R/W state matrix

README.md

@@ -43,7 +43,7 @@ GPU accelerated Neural networks in JavaScript for Browsers and Node.js
     - [For training with NeuralNetwork](#for-training-with-neuralnetwork)
     - [For training with `RNNTimeStep`, `LSTMTimeStep` and `GRUTimeStep`](#for-training-with-rnntimestep-lstmtimestep-and-grutimestep)
     - [For training with `RNN`, `LSTM` and `GRU`](#for-training-with-rnn-lstm-and-gru)
-    - [For training with `AE`](#for-training-with-ae)
+    - [For training with `AutoencoderGPU`](#for-training-with-ae)
   - [Training Options](#training-options)
   - [Async Training](#async-training)
   - [Cross Validation](#cross-validation)
@@ -318,7 +318,7 @@ net.train([
 const output = net.run('I feel great about the world!'); // 'happy'
 ```
 
-#### For training with `AE`
+#### For training with `AutoencoderGPU`
 
 Each training pattern can either:
 
@@ -328,7 +328,7 @@ Each training pattern can either:
 Training an autoencoder to compress the values of a XOR calculation:
 
 ```javascript
-const net = new brain.AE(
+const net = new brain.AutoencoderGPU(
   {
     hiddenLayers: [ 5, 2, 5 ]
   }
@@ -362,8 +362,8 @@ const data = net.denoise(noisyData);
 Test for anomalies in data samples:
 
 ```javascript
-const shouldBeFalse = net.includesAnomalies([0, 1, 1]);
-const shouldBeTrue = net.includesAnomalies([0, 1, 0]);
+const shouldBeFalse = net.likelyIncludesAnomalies([0, 1, 1]);
+const shouldBeTrue = net.likelyIncludesAnomalies([0, 1, 0]);
 ```
 
 ### Training Options
@@ -644,7 +644,7 @@ The user interface used:
 
 - [`brain.NeuralNetwork`](src/neural-network.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation
 - [`brain.NeuralNetworkGPU`](src/neural-network-gpu.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation, GPU version
-- [`brain.AE`](src/autoencoder.ts) - [Autoencoder or "AE"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support
+- [`brain.AutoencoderGPU`](src/autoencoder.ts) - [Autoencoder or "AutoencoderGPU"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support
 - [`brain.recurrent.RNNTimeStep`](src/recurrent/rnn-time-step.ts) - [Time Step Recurrent Neural Network or "RNN"](https://en.wikipedia.org/wiki/Recurrent_neural_network)
 - [`brain.recurrent.LSTMTimeStep`](src/recurrent/lstm-time-step.ts) - [Time Step Long Short Term Memory Neural Network or "LSTM"](https://en.wikipedia.org/wiki/Long_short-term_memory)
 - [`brain.recurrent.GRUTimeStep`](src/recurrent/gru-time-step.ts) - [Time Step Gated Recurrent Unit or "GRU"](https://en.wikipedia.org/wiki/Gated_recurrent_unit)

src/autoencoder-gpu.test.ts

@@ -0,0 +1,76 @@
+import AutoencoderGPU from './autoencoder-gpu';
+import { INeuralNetworkTrainOptions } from './neural-network';
+
+const trainingData = [
+  [0, 0, 0],
+  [0, 1, 1],
+  [1, 0, 1],
+  [1, 1, 0],
+];
+
+const xornet = new AutoencoderGPU<number[], number[]>({
+  inputSize: 3,
+  hiddenLayers: [4, 2, 4],
+  outputSize: 3,
+});
+
+const errorThresh = 0.0011;
+
+const trainOptions: Partial<INeuralNetworkTrainOptions> = {
+  errorThresh,
+  iterations: 250000,
+  learningRate: 0.1,
+  log: (details) => console.log(details),
+  // logPeriod: 500,
+  logPeriod: 500,
+};
+
+const result = xornet.train(trainingData, trainOptions);
+
+test('denoise a data sample', async () => {
+  expect(result.error).toBeLessThanOrEqual(errorThresh);
+
+  function xor(...args: number[]) {
+    return Math.round(xornet.denoise(args)[2]);
+  }
+
+  const run1 = xor(0, 0, 0);
+  const run2 = xor(0, 1, 1);
+  const run3 = xor(1, 0, 1);
+  const run4 = xor(1, 1, 0);
+
+  expect(run1).toBe(0);
+  expect(run2).toBe(1);
+  expect(run3).toBe(1);
+  expect(run4).toBe(0);
+});
+
+test('encode and decode a data sample', async () => {
+  expect(result.error).toBeLessThanOrEqual(errorThresh);
+
+  const run1$input = [0, 0, 0];
+  const run1$encoded = xornet.encode(run1$input);
+  const run1$decoded = xornet.decode(run1$encoded);
+
+  const run2$input = [0, 1, 1];
+  const run2$encoded = xornet.encode(run2$input);
+  const run2$decoded = xornet.decode(run2$encoded);
+
+  for (let i = 0; i < 3; i++)
+    expect(Math.round(run1$decoded[i])).toBe(run1$input[i]);
+  for (let i = 0; i < 3; i++)
+    expect(Math.round(run2$decoded[i])).toBe(run2$input[i]);
+});
+
+test('test a data sample for anomalies', async () => {
+  expect(result.error).toBeLessThanOrEqual(errorThresh);
+
+  function likelyIncludesAnomalies(...args: number[]) {
+    expect(xornet.likelyIncludesAnomalies(args, 0.5)).toBe(false);
+  }
+
+  likelyIncludesAnomalies(0, 0, 0);
+  likelyIncludesAnomalies(0, 1, 1);
+  likelyIncludesAnomalies(1, 0, 1);
+  likelyIncludesAnomalies(1, 1, 0);
+});

src/autoencoder-gpu.ts

@@ -0,0 +1,235 @@
+import {
+  IKernelFunctionThis,
+  KernelOutput,
+  Texture,
+  TextureArrayOutput,
+} from 'gpu.js';
+import {
+  IJSONLayer,
+  INeuralNetworkData,
+  INeuralNetworkDatum,
+  INeuralNetworkTrainOptions,
+  NeuralNetworkIO,
+  NeuralNetworkRAM,
+} from './neural-network';
+import {
+  INeuralNetworkGPUOptions,
+  NeuralNetworkGPU,
+} from './neural-network-gpu';
+import { INeuralNetworkState } from './neural-network-types';
+import { UntrainedNeuralNetworkError } from './errors/untrained-neural-network-error';
+import { DEFAULT_ANOMALY_THRESHOLD } from './autoencoder';
+
+function loss(
+  this: IKernelFunctionThis,
+  actual: number,
+  expected: number,
+  inputs: NeuralNetworkIO,
+  ram: NeuralNetworkRAM
+) {
+  let error = expected - actual;
+
+  // if ( o ≈ i0 ) then return 3.125% of the loss value.
+  // Otherwise, return 3200% of the full loss value.
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-expect-error
+  if (Math.round(actual) !== Math.round(inputs[this.thread.x])) error *= 32;
+  else error *= 0.03125;
+
+  return error;
+}
+
+/**
+ * An autoencoder learns to compress input data down to relevant features and reconstruct input data from its compressed representation.
+ */
+export class AutoencoderGPU<
+  DecodedData extends INeuralNetworkData,
+  EncodedData extends INeuralNetworkData
+> extends NeuralNetworkGPU<DecodedData, DecodedData> {
+  private decoder?: NeuralNetworkGPU<EncodedData, DecodedData>;
+
+  constructor(options?: Partial<INeuralNetworkGPUOptions>) {
+    // Create default options for the autoencoder.
+    options ??= {};
+
+    const decodedSize = options.inputSize ?? options.outputSize ?? 1;
+
+    // Define the denoiser subnet's input and output sizes.
+    options.inputSize = options.outputSize = decodedSize;
+
+    options.hiddenLayers ??= [Math.round(decodedSize * 0.66)];
+
+    options.loss ??= loss;
+
+    // Create the autoencoder.
+    super(options);
+  }
+
+  /**
+   * Denoise input data, removing any anomalies from the data.
+   * @param {DecodedData} input
+   * @returns {DecodedData}
+   */
+  denoise(input: DecodedData): DecodedData {
+    // Run the input through the generic denoiser.
+    // This isn't the best denoiser implementation, but it's efficient.
+    // Efficiency is important here because training should focus on
+    // optimizing for feature extraction as quickly as possible rather than
+    // denoising and anomaly detection; there are other specialized topologies
+    // better suited for these tasks anyways, many of which can be implemented
+    // by using an autoencoder.
+    return this.run(input);
+  }
+
+  /**
+   * Decode `EncodedData` into an approximation of its original form.
+   *
+   * @param {EncodedData} input
+   * @returns {DecodedData}
+   */
+  decode(input: EncodedData): DecodedData {
+    // If the decoder has not been trained yet, throw an error.
+    if (!this.decoder) throw new UntrainedNeuralNetworkError(this);
+
+    // Decode the encoded input.
+    return this.decoder.run(input);
+  }
+
+  /**
+   * Encode data to extract features, reduce dimensionality, etc.
+   *
+   * @param {DecodedData} input
+   * @returns {EncodedData}
+   */
+  encode(input: DecodedData): EncodedData {
+    // If the decoder has not been trained yet, throw an error.
+    if (!this) throw new UntrainedNeuralNetworkError(this);
+
+    // Process the input.
+    this.run(input);
+
+    // Get the auto-encoded input.
+    let encodedInput: TextureArrayOutput = this
+      .encodedLayer as TextureArrayOutput;
+
+    // If the encoded input is a `Texture`, convert it into an `Array`.
+    if (encodedInput instanceof Texture) encodedInput = encodedInput.toArray();
+    else encodedInput = encodedInput.slice(0);
+
+    // Return the encoded input.
+    return encodedInput as EncodedData;
+  }
+
+  /**
+   * Test whether or not a data sample likely contains anomalies.
+   * If anomalies are likely present in the sample, returns `true`.
+   * Otherwise, returns `false`.
+   *
+   * @param {DecodedData} input
+   * @returns {boolean}
+   */
+  likelyIncludesAnomalies(
+    input: DecodedData,
+    anomalyThreshold: number
+  ): boolean {
+    anomalyThreshold ??= DEFAULT_ANOMALY_THRESHOLD;
+    // Create the anomaly vector.
+    const anomalies: number[] = [];
+
+    // Attempt to denoise the input.
+    const denoised = this.denoise(input);
+
+    // Calculate the anomaly vector.
+    for (let i = 0; i < (input.length ?? 0); i++) {
+      anomalies[i] = Math.abs(
+        (input as number[])[i] - (denoised as number[])[i]
+      );
+    }
+
+    // Calculate the sum of all anomalies within the vector.
+    const sum = anomalies.reduce(
+      (previousValue, value) => previousValue + value
+    );
+
+    // Calculate the mean anomaly.
+    const mean = sum / (input as number[]).length;
+
+    // Return whether or not the mean anomaly rate is greater than the anomaly threshold.
+    return mean > anomalyThreshold;
+  }
+
+  /**
+   * Train the auto encoder.
+   *
+   * @param {DecodedData[]} data
+   * @param {Partial<INeuralNetworkTrainOptions>} options
+   * @returns {INeuralNetworkState}
+   */
+  train(
+    data:
+      | Array<Partial<DecodedData>>
+      | Array<INeuralNetworkDatum<Partial<DecodedData>, Partial<DecodedData>>>,
+    options?: Partial<INeuralNetworkTrainOptions>
+  ): INeuralNetworkState {
+    const preprocessedData: Array<INeuralNetworkDatum<
+      Partial<DecodedData>,
+      Partial<DecodedData>
+    >> = [];
+
+    if (data.length && data.length > 0)
+      for (const datum of data) {
+        preprocessedData.push({
+          input: datum as Partial<DecodedData>,
+          output: datum as Partial<DecodedData>,
+        });
+      }
+
+    const results = super.train(preprocessedData, options);
+
+    this.decoder = this.createDecoder();
+
+    return results;
+  }
+
+  /**
+   * Create a new decoder from the trained denoiser.
+   *
+   * @returns {NeuralNetworkGPU<EncodedData, DecodedData>}
+   */
+  private createDecoder() {
+    const json = this.toJSON();
+
+    const layers: IJSONLayer[] = [];
+    const sizes: number[] = [];
+
+    for (let i = this.encodedLayerIndex; i < this.sizes.length; i++) {
+      layers.push(json.layers[i]);
+      sizes.push(json.sizes[i]);
+    }
+
+    json.layers = layers;
+    json.sizes = sizes;
+
+    json.options.inputSize = json.sizes[0];
+
+    const decoder = new NeuralNetworkGPU().fromJSON(json);
+
+    return (decoder as unknown) as NeuralNetworkGPU<EncodedData, DecodedData>;
+  }
+
+  /**
+   * Get the layer containing the encoded representation.
+   */
+  private get encodedLayer(): KernelOutput {
+    return this.outputs[this.encodedLayerIndex];
+  }
+
+  /**
+   * Get the offset of the encoded layer.
+   */
+  private get encodedLayerIndex(): number {
+    return Math.round(this.outputs.length * 0.5) - 1;
+  }
+}
+
+export default AutoencoderGPU;