Well-known types support (Struct, Value)

[mkosieradzki]

protobuf.js version: 6.8.0

I am really surprised that in a library targeting Javascript there is no support for the most important wellknown-types: Struct and Value.

Those types have been specifically designed to allow the best javascript interop.

Struct is mapping to a generic JSON object
Value is mapping to typescript 'any' value

Is this by-design or is it just an oversight?

[dcodeIO]

There is some support for these. For reference: https://github.com/dcodeIO/protobuf.js/blob/master/src/common.js

Still lacking appropriate wrappers, though.

[mkosieradzki]

@dcodeIO Thanks a lot. Any plans for full support (including wrappers) in a predictable future? Or maybe is it something up for grabs?

[xealot]

This would be tremendous. Is there work planned for this?

If you were able to give some pointers on what needs doing perhaps we could open a PR for this support.

[mavrick]

Can we add the protobuf/struct.proto files? would be nice to be able to use them with this lib.

[mailaneel]

This is what we have at the moment

StructWrapper.ts

// tslint:disable-next-line:max-line-length
// @see https://github.com/googleapis/nodejs-common-grpc/blob/67a4cdc109cf3283dbebd487ff672f1fdf3f19bf/src/service.ts

import * as is from 'is';

export class StructEncode {

  seenObjects: Set<{}>;
  removeCircular: boolean;
  stringify?: boolean;

  constructor(options?) {
    // tslint:disable-next-line:no-parameter-reassignment
    options = options || {};

    this.seenObjects = new Set();
    this.removeCircular = options.removeCircular === true;
    this.stringify = options.stringify === true;
  }

  encodeStruct(obj) {
    const convertedObject = {
      fields: {},
    };

    this.seenObjects.add(obj);

    for (const prop in obj) {
      if (obj.hasOwnProperty(prop)) {
        const value = obj[prop];

        if (is.undefined(value)) {
          continue;
        }

        convertedObject.fields[prop] = this.encodeValue(value);
      }
    }

    this.seenObjects.delete(obj);

    return convertedObject;
  }

  encodeValue(value) {
    let convertedValue;

    if (is.null(value)) {
      convertedValue = {
        nullValue: 0,
      };
    } else if (is.number(value)) {
      convertedValue = {
        numberValue: value,
      };
    } else if (is.string(value)) {
      convertedValue = {
        stringValue: value,
      };
    } else if (is.boolean(value)) {
      convertedValue = {
        boolValue: value,
      };
    } else if (Buffer.isBuffer(value)) {
      convertedValue = {
        blobValue: value,
      };
    } else if (is.object(value)) {
      if (this.seenObjects.has(value)) {
        // Circular reference.
        if (!this.removeCircular) {
          throw new Error(
            [
              'This object contains a circular reference. To automatically',
              'remove it, set the `removeCircular` option to true.',
            ].join(' ')
          );
        }
        convertedValue = {
          stringValue: '[Circular]',
        };
      } else {
        convertedValue = {
          structValue: this.encodeStruct(value),
        };
      }
    } else if (is.array(value)) {
      convertedValue = {
        listValue: {
          values: value.map(this.encodeValue.bind(this)),
        },
      };
    } else {
      if (!this.stringify) {
        throw new Error('Value of type ' + typeof value + ' not recognized.');
      }

      convertedValue = {
        stringValue: String(value),
      };
    }

    return convertedValue;
  }
}

export class StructDecode {

  static decodeValue(value) {
    switch (value.kind) {
      case 'structValue': {
        return StructDecode.decodeStruct(value.structValue);
      }

      case 'nullValue': {
        return null;
      }

      case 'listValue': {
        return value.listValue.values.map(StructDecode.decodeValue);
      }

      default: {
        return value[value.kind];
      }
    }
  }

  static decodeStruct(struct) {
    const convertedObject = {};

    for (const prop in struct.fields) {
      if (struct.fields.hasOwnProperty(prop)) {
        const value = struct.fields[prop];
        convertedObject[prop] = StructDecode.decodeValue(value);
      }
    }

    return convertedObject;
  }

}

wrappers.ts

import { wrappers } from 'protobufjs';
import { StructDecode, StructEncode } from './StructWrapper';

wrappers['.google.protobuf.Value'] = <any>{
  fromObject(object) {
    if (object) {
      return (new StructEncode()).encodeValue(object);
    }

    return this.fromObject(object);
  },

  toObject(message: any) {
    return StructDecode.decodeValue(message);
  }
};

wrappers['.google.protobuf.Struct'] = <any>{
  fromObject(object) {
    if (object) {
      return (new StructEncode()).encodeStruct(object);
    }

    return this.fromObject(object);
  },

  toObject(message: any) {
    return StructDecode.decodeStruct(message);
  }
};

[kiranmantri]

Any plans on when this Struct, Any and other will be available ?

[Paic]

Any news on this issue since the last comment ?

[guyisra]

any updates?

[shizhx]

any updates?
is this repo dead?

[gebv]

Use static method fromJavaScript for Struct
See
https://github.com/protocolbuffers/protobuf/blob/4d6712e73995e0c64eb5f208e7388f824175b3b8/js/proto3_test.js#L466

For me works

If need get Value follow the code (dirty solution?)

import { Struct } from "google-protobuf/google/protobuf/struct_pb";

var jsObj = {
          abc: "def",
          number: 12345.678,
          nullKey: null,
          boolKey: true,
          listKey: [1, null, true, false, "abc"],
          structKey: {foo: "bar", somenum: 123},
          complicatedKey: [{xyz: {abc: [3, 4, null, false]}}, "zzz"]
        };
Struct.fromJavaScript({val: jsObj}).getFieldsMap().get('val')

[classLfz]

same issue here, I read the official document about struct , then I write this func to build the struct data to protobuf:

function buildGoogleStructValue (val, sub = false) {
  const typeofVal = typeof val
  const baseValueTypes = {
    number: 'numberValue',
    string: 'stringValue',
    boolean: 'boolValue'
  }
  if (Object.keys(baseValueTypes).includes(typeofVal)) {
    return {
      [baseValueTypes[typeofVal]]: val
    }
  }
  if (Array.isArray(val)) {
    const out = {
      listValue: {
        values: []
      }
    }
    val.forEach(valItem => {
      const itemVal = buildGoogleStructValue(valItem, true)
      out.listValue.values.push(itemVal)
    })
    return out
  }
  if (typeofVal === 'object') {
    const out = sub ? {
      structValue: {
        fields: {}
      }
    } : {
      fields: {}
    }
    Object.keys(val).forEach(field => {
      if (sub) {
        out.structValue.fields[field] = buildGoogleStructValue(val[field], true)
      } else {
        out.fields[field] = buildGoogleStructValue(val[field], true)
      }
    })
    return out
  }
}

proto:

message Message {
    google.protobuf.Struct struct = 1;
}

so, I can build message data like this:

const message = {
    struct: buildGoogleStructValue({
        string: '1',
        bool: true,
        number: 12,
        struct: {
            structField1: 1000
        },
        list: [1, '12']
    }
})

It's worked for me.

[devhossamali]

Use Struct method fromJavaScript

const obj = {somekey: 'foo'};
const result = Struct.fromJavaScript(obj);

and pass result to the message setter for the Struct field

[vokilam-d]

@classLfz, thank you, your solution worked great for me! Only this I added support for null value.

Also, I created deserialization function (to use in client), based on @classLfz's serialization. If anyone is interested, here's full code:

const isObject = (obj: any): boolean => typeof obj === 'object' && !Array.isArray(obj) && obj !== null;

enum FieldName {
  Number = 'numberValue',
  String = 'stringValue',
  Boolean = 'boolValue',
  Null = 'nullValue',
  List = 'listValue',
  Struct = 'structValue',
}

const typeofFieldNameMap = {
  number: FieldName.Number,
  string: FieldName.String,
  boolean: FieldName.Boolean,
}

const baseFieldNameConstructorMap = {
  [FieldName.Number]: Number,
  [FieldName.String]: String,
  [FieldName.Boolean]: Boolean,
}

const nullFieldValue = 0;

export const serializeGoogleStructValue = (val: any, sub = false) => {
  if (val === null || val === undefined) {
    return {
      [FieldName.Null]: nullFieldValue
    };
  }

  const typeofVal = typeof val;
  if (Object.keys(typeofFieldNameMap).includes(typeofVal)) {
    return {
      [typeofFieldNameMap[typeofVal]]: val
    };
  }
  if (Array.isArray(val)) {
    const out = {
      [FieldName.List]: {
        values: []
      }
    };
    for (const valItem of val) {
      const itemVal = serializeGoogleStructValue(valItem, true);
      out[FieldName.List].values.push(itemVal);
    }
    return out
  }
  if (typeofVal === 'object') {
    const out = sub ? {
      [FieldName.Struct]: {
        fields: {}
      }
    } : {
      fields: {}
    }
    for (const field of Object.keys(val)) {
      if (val[field] === undefined) {
        continue;
      }

      if (sub) {
        out[FieldName.Struct].fields[field] = serializeGoogleStructValue(val[field], true);
      } else {
        out.fields[field] = serializeGoogleStructValue(val[field], true);
      }
    }

    return out;
  }
}

export const deserializeGoogleStructValue = (val: any, sub = false) => {
  if (sub === false && !isObject(val?.fields)) {
    throw new Error(`Invalid Struct format. Object must include "fields" property`);
  }

  const fieldName = Object.keys(val)[0];
  if (fieldName === FieldName.Null) {
    return null;
  }

  const baseValueTypeConstructor = baseFieldNameConstructorMap[fieldName];
  if (baseValueTypeConstructor) {
    return baseValueTypeConstructor(val[fieldName]);
  }

  if (fieldName === FieldName.List) {
    return val[fieldName].values.map(listValue => deserializeGoogleStructValue(listValue, true));
  }

  if (fieldName === FieldName.Struct) {
    return deserializeGoogleStructValue(val[fieldName], true);
  }

  if (isObject(val.fields)) {
    const result = {};
    Object.keys(val.fields).forEach(fieldName => {
      result[fieldName] = deserializeGoogleStructValue(val.fields[fieldName], true);
    });
    return result;
  }
}