Storage: Storage.writeJSON now escapes with a more compact \x##/`\#…

…` character escape notation (still valid JSON)

              This allows Espruino to save non-unicode strings with characters in the unicode range, and to also re-load them as non-unicode

ChangeLog

@@ -26,6 +26,8 @@
             Graphics: Improve PBF font loading to handle v3, plus Espruino extension to handle >10k glyphs in one file
             Graphics: Graphics.stringMetrics now returns 'unknownChars' to indicate if a font can't render a character in the String
             Fix unicode in object accesses, eg c["\u00FC"]=42 (fix #2429)
+            Storage: Storage.writeJSON now escapes with a more compact `\x##`/`\#` character escape notation (still valid JSON)
+              This allows Espruino to save non-unicode strings with characters in the unicode range, and to also re-load them as non-unicode
 
      2v19 : Fix Object.values/entries for numeric keys after 2v18 regression (fix #2375)
             nRF52: for SD>5 use static buffers for advertising and scan response data (#2367)

src/jsutils.c

@@ -752,8 +752,8 @@ JsVarFloat wrapAround(JsVarFloat val, JsVarFloat size) {
  * * `%s` = string (char *)
  * * `%c` = char
  * * `%v` = JsVar * (doesn't have to be a string - it'll be converted)
- * * `%q` = JsVar * (in quotes, and escaped)
- * * `%Q` = JsVar * (in quotes, and escaped the JSON subset of escape chars)
+ * * `%q` = JsVar * (in quotes, and escaped with \uXXXX,\xXX,\X whichever makes sense)
+ * * `%Q` = JsVar * (in quotes, and escaped with only \uXXXX)
  * * `%j` = Variable printed as JSON
  * * `%t` = Type of variable
  * * `%p` = Pin
@@ -824,6 +824,7 @@ void vcbprintf(
         bool isJSONStyle = fmtChar=='Q';
         if (quoted) user_callback("\"",user_data);
         JsVar *v = jsvAsString(va_arg(argp, JsVar*));
+        if (jsvIsUTF8String(v)) isJSONStyle=true; // if it's a UTF8 string make sure we escape in UTF8 form to force Espruino to re-create it as a UTF8 string when parsing
         buf[1] = 0;
         if (jsvIsString(v)) {
           JsvStringIterator it;

src/jsutils.h

@@ -581,8 +581,8 @@ typedef void (*vcbprintf_callback)(const char *str, void *user_data);
  * * `%s` = string (char *)
  * * `%c` = char
  * * `%v` = JsVar * (doesn't have to be a string - it'll be converted)
- * * `%q` = JsVar * (in quotes, and escaped)
- * * `%Q` = JsVar * (in quotes, and escaped the JSON subset of escape chars)
+ * * `%q` = JsVar * (in quotes, and escaped with \uXXXX,\xXX,\X whichever makes sense)
+ * * `%Q` = JsVar * (in quotes, and escaped with only \uXXXX)
  * * `%j` = Variable printed as JSON
  * * `%t` = Type of variable
  * * `%p` = Pin

src/jswrap_json.c

@@ -293,7 +293,7 @@ static bool jsfGetJSONForObjectItWithCallback(JsvObjectIterator *it, JSONFlags f
           if (isIDString(buf)) addQuotes=false;
         }
       }
-      cbprintf(user_callback, user_data, addQuotes?((flags&JSON_JSON_COMPATIBILE)?"%Q%s":"%q%s"):"%v%s", index, (flags&JSON_PRETTY)?": ":":");
+      cbprintf(user_callback, user_data, addQuotes?((flags&JSON_ALL_UNICODE_ESCAPE)?"%Q%s":"%q%s"):"%v%s", index, (flags&JSON_PRETTY)?": ":":");
       if (first)
         first = false;
       jsfGetJSONWithCallback(item, index, nflags, whitespace, user_callback, user_data);
@@ -482,17 +482,17 @@ void jsfGetJSONWithCallback(JsVar *var, JsVar *varName, JSONFlags flags, const c
       cbprintf(user_callback, user_data, "function ");
       jsfGetJSONForFunctionWithCallback(var, nflags, user_callback, user_data);
     }
-  } else if ((jsvIsString(var) && !jsvIsName(var)) || ((flags&JSON_JSON_COMPATIBILE)&&jsvIsPin(var))) {
+  } else if ((jsvIsString(var) && !jsvIsName(var)) || ((flags&JSON_PIN_TO_STRING)&&jsvIsPin(var))) {
     if ((flags&JSON_LIMIT) && jsvGetStringLength(var)>JSON_LIMIT_STRING_AMOUNT) {
       // if the string is too big, split it and put dots in the middle
       JsVar *var1 = jsvNewFromStringVar(var, 0, JSON_LIMITED_STRING_AMOUNT);
       JsVar *var2 = jsvNewFromStringVar(var, jsvGetStringLength(var)-JSON_LIMITED_STRING_AMOUNT, JSON_LIMITED_STRING_AMOUNT);
       cbprintf(user_callback, user_data, "%q%s%q", var1, JSON_LIMIT_TEXT, var2);
       jsvUnLock2(var1, var2);
     } else {
-      cbprintf(user_callback, user_data, (flags&JSON_JSON_COMPATIBILE)?"%Q":"%q", var);
+      cbprintf(user_callback, user_data, (flags&JSON_ALL_UNICODE_ESCAPE)?"%Q":"%q", var);
     }
-  } else if ((flags&JSON_JSON_COMPATIBILE) && jsvIsFloat(var) && !isfinite(jsvGetFloat(var))) {
+  } else if ((flags&JSON_NO_NAN) && jsvIsFloat(var) && !isfinite(jsvGetFloat(var))) {
     cbprintf(user_callback, user_data, "null");
   } else {
     cbprintf(user_callback, user_data, "%v", var);

src/jswrap_json.h

@@ -32,14 +32,13 @@ typedef enum {
   JSON_ARRAYBUFFER_AS_ARRAY = 128, //< dump arraybuffers as arrays
   JSON_SHOW_OBJECT_NAMES    = 256, //< Show 'Promise {}'/etc for objects if the type is global
   JSON_DROP_QUOTES          = 512, //< When outputting objects, drop quotes for alphanumeric field names
-  JSON_JSON_COMPATIBILE     = 1024, /**<
-    Only use unicode for escape characters - needed for JSON compatibility
-    Don't output NaN for NaN numbers, only 'null'
-    Convert pins to Strings
-  */
-  JSON_ALLOW_TOJSON      = 2048, //< If there's a .toJSON function in an object, use it and parse that
+  JSON_PIN_TO_STRING        = 1024, //< Convert pins to Strings
+  JSON_ALL_UNICODE_ESCAPE   = 2048, //< Only use unicode \xXXXX for escape characters, not \xXX or \X
+  JSON_NO_NAN               = 4096, //< Don't output NaN for NaN numbers, only 'null'
+  JSON_JSON_COMPATIBILE     = JSON_PIN_TO_STRING|JSON_ALL_UNICODE_ESCAPE|JSON_NO_NAN, //< specific stuff needed for compatibility
+  JSON_ALLOW_TOJSON      = 8192, //< If there's a .toJSON function in an object, use it and parse that
   // ...
-  JSON_INDENT            = 4096, // MUST BE THE LAST ENTRY IN JSONFlags - we use this to count the amount of indents
+  JSON_INDENT            = 16384, // MUST BE THE LAST ENTRY IN JSONFlags - we use this to count the amount of indents
 } JSONFlags;
 
 /* This is like jsfGetJSONWithCallback, but handles ONLY functions (and does not print the initial 'function' text) */

src/jswrap_storage.c

@@ -285,13 +285,23 @@ disappear when the device resets or power is lost.
 Simply write `require("Storage").writeJSON("MyFile", [1,2,3])` to write a new
 file, and `require("Storage").readJSON("MyFile")` to read it.
 
-This is equivalent to: `require("Storage").write(name, JSON.stringify(data))`
+This is (almost) equivalent to: `require("Storage").write(name, JSON.stringify(data))`
 
 **Note:** This function should be used with normal files, and not `StorageFile`s
 created with `require("Storage").open(filename, ...)`
+
+**Note:** Normally `JSON.stringify` converts any non-standard character to an escape code with `\uXXXX`, but
+as of Espruino 2v20, when writing to a file we use the most compact form, like `\xXX` or `\X`. This saves
+space and is faster, but also means that if a String wasn't a UTF8 string but contained characters in the UTF8 codepoint range,
+when saved it won't end up getting reloaded as a UTF8 string.
 */
 bool jswrap_storage_writeJSON(JsVar *name, JsVar *data) {
-  JsVar *d = jswrap_json_stringify(data,0,0);
+  JsVar *d = jsvNewFromEmptyString();
+  if (!d) return false;
+  /* Don't call jswrap_json_stringify directly because we want to ensure we don't use JSON_JSON_COMPATIBILE, so
+  String escapes like `\xFC` stay as `\xFC` and not `\u00FC` to save space and help with unicode compatibility
+  */
+  jsfGetJSON(data, d, (JSON_IGNORE_FUNCTIONS|JSON_NO_UNDEFINED|JSON_ARRAYBUFFER_AS_ARRAY|JSON_JSON_COMPATIBILE) &~JSON_ALL_UNICODE_ESCAPE);
   bool r = jsfWriteFile(jsfNameFromVar(name), d, JSFF_NONE, 0, 0);
   jsvUnLock(d);
   return r;
@@ -426,7 +436,7 @@ void jswrap_storage_debug() {
   "name" : "getFree",
   "params" : [
     ["checkInternalFlash","bool","Check the internal flash (rather than external SPI flash).  Default false, so will check external storage"]
-  ],  
+  ],
   "generate" : "jswrap_storage_getFree",
   "return" : ["int","The amount of free bytes"]
 }
@@ -451,7 +461,7 @@ int jswrap_storage_getFree(bool checkInternalFlash) {
   "name" : "getStats",
   "params" : [
     ["checkInternalFlash","bool","Check the internal flash (rather than external SPI flash).  Default false, so will check external storage"]
-  ],  
+  ],
   "generate" : "jswrap_storage_getStats",
   "return" : ["JsVar","An object containing info about the current Storage system"]
 }
@@ -476,8 +486,8 @@ JsVar *jswrap_storage_getStats(bool checkInternalFlash) {
   uint32_t addr = 0;
 #ifdef FLASH_SAVED_CODE2_START
   addr = checkInternalFlash ? FLASH_SAVED_CODE_START : FLASH_SAVED_CODE2_START;
-  
-#endif  
+
+#endif
   JsfStorageStats stats = jsfGetStorageStats(addr, true);
   jsvObjectSetChildAndUnLock(o, "totalBytes", jsvNewFromInteger((JsVarInt)stats.total));
   jsvObjectSetChildAndUnLock(o, "freeBytes", jsvNewFromInteger((JsVarInt)stats.free));