refactor(v2): serialization (#6916)

* fix some signal deserialization

* fix vnode serialize

* fix store

* resource working

* fix(ssr): await close container

* Date: support invalid+use shorter epoch time

* add setter for deserialized object

* add test for setting a value

* fix qrl scope deduping

* fix(nix): playwright running

---------

Co-authored-by: Varixo <[email protected]>

package.json

@@ -102,7 +102,7 @@
     "@napi-rs/triples": "1.2.0",
     "@node-rs/helper": "1.6.0",
     "@octokit/action": "6.1.0",
-    "@playwright/test": "1.47.2",
+    "@playwright/test": "1.47.0",
     "@types/brotli": "1.3.4",
     "@types/bun": "1.1.6",
     "@types/cross-spawn": "6.0.6",

packages/qwik/src/core/client/dom-container.ts

@@ -339,8 +339,8 @@ export class DomContainer extends _SharedContainer implements IClientContainer {
       id = parseFloat(id);
     }
     assertTrue(
-      id < this.$rawStateData$.length,
-      `Invalid reference: ${id} < ${this.$rawStateData$.length}`
+      id < this.$rawStateData$.length / 2,
+      `Invalid reference: ${id} >= ${this.$rawStateData$.length / 2}`
     );
     return this.stateData[id];
   };

packages/qwik/src/core/shared/qrl/qrl-class.ts

@@ -73,7 +73,7 @@ export const createQRL = <TYPE>(
   symbol: string,
   symbolRef: null | ValueOrPromise<TYPE>,
   symbolFn: null | (() => Promise<Record<string, TYPE>>),
-  capture: null | Readonly<string[]>,
+  capture: null | Readonly<number[]>,
   captureRef: Readonly<unknown[]> | null,
   refSymbol: string | null
 ): QRLInternal<TYPE> => {

packages/qwik/src/core/shared/qrl/qrl.unit.ts

@@ -62,29 +62,29 @@ describe('serialization', () => {
       $symbol$: 's1',
       $capture$: null,
     });
-    matchProps(parseQRL('./chunk#s1[1 b]'), {
+    matchProps(parseQRL('./chunk#s1[1 2]'), {
       $chunk$: './chunk',
       $symbol$: 's1',
-      $capture$: ['1', 'b'],
+      $capture$: [1, 2],
     });
-    matchProps(parseQRL('./chunk#s1[1 b]'), {
+    matchProps(parseQRL('./chunk#s1[1 2]'), {
       $chunk$: './chunk',
       $symbol$: 's1',
-      $capture$: ['1', 'b'],
+      $capture$: [1, 2],
     });
-    matchProps(parseQRL('./chunk#s1[1 b]'), {
+    matchProps(parseQRL('./chunk#s1[1 2]'), {
       $chunk$: './chunk',
       $symbol$: 's1',
-      $capture$: ['1', 'b'],
+      $capture$: [1, 2],
     });
-    matchProps(parseQRL('./chunk[1 b]'), {
+    matchProps(parseQRL('./chunk[1 2]'), {
       $chunk$: './chunk',
-      $capture$: ['1', 'b'],
+      $capture$: [1, 2],
     });
     matchProps(parseQRL('./path#symbol[2]'), {
       $chunk$: './path',
       $symbol$: 'symbol',
-      $capture$: ['2'],
+      $capture$: [2],
     });
     matchProps(
       parseQRL(
@@ -93,7 +93,7 @@ describe('serialization', () => {
       {
         $chunk$: '/src/path%2d/foo_symbol.js?_qrl_parent=/home/user/project/src/path/foo.js',
         $symbol$: 'symbol',
-        $capture$: ['2'],
+        $capture$: [2],
       }
     );
   });
@@ -102,6 +102,7 @@ describe('serialization', () => {
     const serializationContext = createSerializationContext(
       null,
       () => '',
+      () => '',
       () => {}
     );
     assert.equal(

packages/qwik/src/core/shared/serialization.md

@@ -0,0 +1,39 @@
+# State Serialization
+
+The state is stored as an array of values, called "roots". These roots are either added before serialization starts, or during the serialization.
+Some are added during serialization to be able to refer to them from multiple places.
+
+The values are serialized in the following format:
+
+- Even values are always TypeIds, specifying the type of the next value.
+- Odd values are the encoded actual values.
+  - Then encoded values can only be numbers, strings or arrays
+  - Arrays are used to store more complex metadata. Prefer these over encoding data into strings.
+  - If a typeId is `undefined`, that means it's been restored already and the value is "raw"
+- Array encoded values use the same encoding
+
+There are various supported types, but one that is important is the reference type. It refers to a state root by its index. Because of the encoding, the actual data for the state root will be at `(index*2, index*2 + 1)`.
+
+## Serializing
+
+First, all the state roots are walked and awaited to identify objects that are referred to multiple times (including cycles). Any such objects that are added as roots too. This happens in `breakCircularDependenciesAndResolvePromises`.
+
+Then the roots are serialized one by one, and the result is a text stream with occurrences of `</` escaped to prevent injection attacks.
+
+## Restoring
+
+To restore, we use a proxy that will lazily recreate the values.
+Restoring a value happens in two steps:
+
+- Allocate: The value is created, but not filled in. The value is stored.
+- Inflate: The value is inflated by walking the object graph and filling in the values. Reference cycles will find the value as it is being inflated.
+
+This two-step approach is used to support circular dependencies. By first creating the empty object you can store its reference, which can then already be used while filling in the values. (this is how ESM imports work too)
+
+### Lazy restore
+
+To avoid blocking the main thread on wake, we lazily restore the roots, with caching.
+
+The serialized text is first parsed to get an array of encoded root data.
+
+Then, a proxy gets the raw data and returns an array that deserializes properties on demand and caches them. Objects are also lazily restored.

packages/qwik/src/core/shared/shared-container.ts

@@ -53,6 +53,7 @@ export abstract class _SharedContainer implements Container {
     return createSerializationContext(
       NodeConstructor,
       symbolToChunkResolver,
+      this.getHostProp.bind(this),
       this.setHostProp.bind(this),
       writer
     );