Fix <ins>/<del> with structured headers (#533)

src/Clause.ts

@@ -13,7 +13,7 @@ import {
   parseH1,
   ParsedHeader,
 } from './header-parser';
-import { offsetToLineAndColumn } from './utils';
+import { offsetToLineAndColumn, traverseWhile } from './utils';
 
 const aoidTypes = [
   'abstract operation',
@@ -32,7 +32,11 @@ export const SPECIAL_KINDS_MAP = new Map([
 export const SPECIAL_KINDS = [...SPECIAL_KINDS_MAP.keys()];
 
 export function extractStructuredHeader(header: Element): Element | null {
-  const dl = header.nextElementSibling;
+  const dl = traverseWhile(
+    header.nextElementSibling,
+    'nextElementSibling',
+    el => el.nodeName === 'DEL'
+  );
   if (dl == null || dl.tagName !== 'DL' || !dl.classList.contains('header')) {
     return null;
   }
@@ -95,39 +99,43 @@ export default class Clause extends Builder {
     }
 
     this.signature = null;
-    let header = this.node.firstElementChild;
-    while (header != null && header.tagName === 'SPAN' && header.children.length === 0) {
-      // skip oldids
-      header = header.nextElementSibling;
-    }
-    if (header == null) {
+    const header = traverseWhile(
+      this.node.firstElementChild,
+      'nextElementSibling',
+      // skip <del> and oldids
+      el => el.nodeName === 'DEL' || (el.nodeName === 'SPAN' && el.children.length === 0)
+    );
+    let headerH1 = traverseWhile(header, 'firstElementChild', el => el.nodeName === 'INS', {
+      once: true,
+    });
+    if (headerH1 == null) {
       this.spec.warn({
         type: 'node',
         ruleId: 'missing-header',
         message: `could not locate header element`,
         node: this.node,
       });
-      header = null;
-    } else if (header.tagName !== 'H1') {
+      headerH1 = null;
+    } else if (headerH1.tagName !== 'H1') {
       this.spec.warn({
         type: 'node',
         ruleId: 'missing-header',
-        message: `could not locate header element; found <${header.tagName.toLowerCase()}> before any <h1>`,
-        node: header,
+        message: `could not locate header element; found <${header!.tagName.toLowerCase()}> before any <h1>`,
+        node: header!,
       });
-      header = null;
+      headerH1 = null;
     } else {
-      this.buildStructuredHeader(header);
+      this.buildStructuredHeader(headerH1, header!);
     }
-    this.header = header;
-    if (header == null) {
+    this.header = headerH1;
+    if (headerH1 == null) {
       this.title = 'UNKNOWN';
       this.titleHTML = 'UNKNOWN';
     }
   }
 
-  buildStructuredHeader(header: Element) {
-    const dl = extractStructuredHeader(header);
+  buildStructuredHeader(header: Element, headerSurrogate: Element = header) {
+    const dl = extractStructuredHeader(headerSurrogate);
     if (dl === null) {
       return;
     }

src/Production.ts

@@ -52,18 +52,14 @@ export default class Production extends Builder {
     if (node.hasAttribute('primary')) {
       primary = true;
     } else {
-      let parent = node.parentElement;
-      if (parent != null) {
+      const parent = utils.traverseWhile(
+        node.parentElement,
+        'parentElement',
         // highlighted nodes still count as primary unless they are being deleted (i.e. in a <del> tag)
-        while (parent.tagName === 'INS' || parent.tagName === 'MARK') {
-          parent = parent.parentElement;
-          if (parent == null) {
-            break;
-          }
-        }
-        if (parent != null && parent.tagName === 'EMU-GRAMMAR') {
-          primary = parent.hasAttribute('primary') || parent.getAttribute('type') === 'definition';
-        }
+        el => el.nodeName === 'INS' || el.nodeName === 'MARK'
+      );
+      if (parent != null && parent.tagName === 'EMU-GRAMMAR') {
+        primary = parent.hasAttribute('primary') || parent.getAttribute('type') === 'definition';
       }
     }
     this.primary = primary;

src/clauseNums.ts

@@ -108,17 +108,21 @@ export default function iterator(spec: Spec): ClauseNumberIterator {
             message:
               'multi-step explicit clause numbers should not be mixed with single-step clause numbers in the same parent clause',
           });
-        } else if (
-          nums.some((n, i) => n < ids[level][i]) ||
-          nums.every((n, i) => n === ids[level][i])
-        ) {
-          spec.warn({
-            type: 'attr-value',
-            node,
-            attr: 'number',
-            ruleId: 'invalid-clause-number',
-            message: 'clause numbers should be strictly increasing',
-          });
+        } else {
+          // Make sure that `nums` is strictly greater than `ids[level]` (i.e.,
+          // that their items are not identical and that the item in `nums` is
+          // strictly greater than the value in `ids[level]` at the first
+          // index where they differ).
+          const i = nums.findIndex((num, i) => num !== ids[level][i]);
+          if (i < 0 || !(nums[i] > ids[level][i])) {
+            spec.warn({
+              type: 'attr-value',
+              node,
+              attr: 'number',
+              ruleId: 'invalid-clause-number',
+              message: 'clause numbers should be strictly increasing',
+            });
+          }
         }
       }
       return nums;

src/header-parser.ts

@@ -237,7 +237,7 @@ export function parseH1(headerText: string): ParsedHeaderOrFailure {
   if (match) {
     offset += match[0].length;
     returnOffset = offset;
-    ({ match, text } = eat(text, /^(.*)(?!<\/(ins|del|mark)>)/i));
+    ({ match, text } = eat(text, /^(.*?)(?=<\/(ins|del|mark)>|$)/im));
     if (match) {
       returnType = match[1].trim();
       if (returnType === '') {

src/utils.ts

@@ -117,6 +117,21 @@ export function replaceTextNode(node: Node, frag: DocumentFragment) {
   return newXrefNodes;
 }
 
+/*@internal*/
+export function traverseWhile<P extends string, T extends Record<P, T | null>>(
+  node: T | null,
+  relationship: P,
+  predicate: (node: T) => boolean,
+  options?: { once?: boolean }
+): T | null {
+  const once = options?.once ?? false;
+  while (node != null && predicate(node)) {
+    node = node[relationship];
+    if (once) break;
+  }
+  return node;
+}
+
 /*@internal*/
 export function logVerbose(str: string) {
   const dateString = new Date().toISOString();
@@ -132,20 +147,12 @@ export function logWarning(str: string) {
 const CLAUSE_LIKE = ['EMU-ANNEX', 'EMU-CLAUSE', 'EMU-INTRO', 'EMU-NOTE', 'BODY'];
 /*@internal*/
 export function shouldInline(node: Node) {
-  let parent = node.parentNode;
+  const surrogateParentTags = ['EMU-GRAMMAR', 'EMU-IMPORT', 'INS', 'DEL'];
+  const parent = traverseWhile(node.parentNode, 'parentNode', node =>
+    surrogateParentTags.includes(node?.nodeName ?? '')
+  );
   if (!parent) return false;
 
-  while (
-    parent &&
-    parent.parentNode &&
-    (parent.nodeName === 'EMU-GRAMMAR' ||
-      parent.nodeName === 'EMU-IMPORT' ||
-      parent.nodeName === 'INS' ||
-      parent.nodeName === 'DEL')
-  ) {
-    parent = parent.parentNode;
-  }
-
   const clauseLikeParent =
     CLAUSE_LIKE.includes(parent.nodeName) ||
     CLAUSE_LIKE.includes((parent as Element).getAttribute('data-simulate-tagname')?.toUpperCase()!);

test/baselines.js

@@ -35,7 +35,8 @@ describe('baselines', () => {
   let optionsSets = [{ lintSpec: false }];
   for (let file of files) {
     const reference = REFERENCE_DIR + file;
-    it(SOURCES_DIR + file, async () => {
+    const sourcePath = SOURCES_DIR + file;
+    it(sourcePath, async () => {
       let expectedFiles = new Map();
 
       (function walk(f) {
@@ -51,7 +52,7 @@ describe('baselines', () => {
         }
       })(reference);
 
-      let spec = await build(SOURCES_DIR + file, {});
+      let spec = await build(sourcePath, {});
 
       let actualFiles = handleSingleFileOutput(spec.generatedFiles);
 
@@ -81,14 +82,39 @@ describe('baselines', () => {
       if (rebaseline) {
         return;
       }
+
+      let contents = fs.readFileSync(sourcePath, 'utf8');
+      let expectedWarnings = [...contents.matchAll(/<!--\s+EXPECT_WARNING(.*?)-->/g)].map(m =>
+        JSON.parse(m[1])
+      );
+      let warningProps = new Set(expectedWarnings.flatMap(obj => Object.keys(obj)));
+      function pickFromWarning(warning) {
+        if (warningProps.size === 0) {
+          // No warnings are expected, so "pick" the entire object.
+          return warning;
+        }
+        let picks = {};
+        for (let [key, value] of Object.entries(warning)) {
+          if (warningProps.has(key)) picks[key] = value;
+        }
+        return picks;
+      }
+
       for (let options of optionsSets) {
-        let spec = await build(SOURCES_DIR + file, options);
+        let warnings = [];
+        let warn = warning => warnings.push(warning);
+        let spec = await build(sourcePath, { ...options, warn });
         let actualFiles = handleSingleFileOutput(spec.generatedFiles);
         assert.deepStrictEqual(
           actualFiles,
           expectedFiles,
           `output differed when using option ${JSON.stringify(options)}`
         );
+        assert.deepStrictEqual(
+          warnings.map(warning => pickFromWarning(warning)),
+          expectedWarnings,
+          `warnings differed when using option ${JSON.stringify(options)}`
+        );
       }
     });
   }

test/baselines/generated-reference/clauses.html

@@ -6,7 +6,7 @@
 
   <li><span>Jump to search box</span><code>/</code></li>
 </ul></div><div id="spec-container">
-<div><h2>Table of Contents</h2><ol class="toc"><li><a href="#sec-intro" title="Intro">Intro</a><ol class="toc"><li><a href="#sec-intro2" title="Sub Intro">Sub Intro</a></li></ol></li><li><a href="#sec-clause" title="Clause Foo(a, b)"><span class="secnum">1</span> Clause Foo(<var>a</var>, <var>b</var>)</a><ol class="toc"><li><a href="#Foo" title="Sub Clause"><span class="secnum">1.1</span> Sub Clause</a></li></ol></li><li><a href="#sec-number" title="Explicit number"><span class="secnum">7</span> Explicit number</a><ol class="toc"><li><a href="#sec-number1" title="Automatic number inside explicit number"><span class="secnum">7.1</span> Automatic number inside explicit number</a></li><li><a href="#sec-number2" title="Nested explicit number"><span class="secnum">7.4</span> Nested explicit number</a></li><li><a href="#sec-number3" title="Automatic number after explicit number"><span class="secnum">7.5</span> Automatic number after explicit number</a></li><li><a href="#sec-number-nested" title="Multi-step explicit numbers"><span class="secnum">7.6</span> Multi-step explicit numbers</a><ol class="toc"><li><a href="#sec-number-multi-step" title="Multi-step explicit numbers"><span class="secnum">7.6.1.1</span> Multi-step explicit numbers</a></li><li><a href="#sec-number-multi-step-1" title="Automatic number after explicit number"><span class="secnum">7.6.1.2</span> Automatic number after explicit number</a><ol class="toc"><li><a href="#sec-number-multi-step-inner" title="Nested clause after explicit multi-step number"><span class="secnum">7.6.1.2.1</span> Nested clause after explicit multi-step number</a></li></ol></li><li><a href="#sec-number-multi-step" title="Increasing multi-step explicit numbers"><span class="secnum">7.6.2.1</span> Increasing multi-step explicit numbers</a></li></ol></li></ol></li><li><a href="#sec-annex" title="Annex"><span class="secnum">A</span> Annex</a><ol class="toc"><li><a href="#sec-annex2" title="Sub-annex"><span class="secnum">A.1</span> Sub-annex</a></li></ol></li></ol></div><emu-intro id="sec-intro">
+<div><h2>Table of Contents</h2><ol class="toc"><li><a href="#sec-intro" title="Intro">Intro</a><ol class="toc"><li><a href="#sec-intro2" title="Sub Intro">Sub Intro</a></li></ol></li><li><a href="#sec-clause" title="Clause Foo(a, b)"><span class="secnum">1</span> Clause Foo(<var>a</var>, <var>b</var>)</a><ol class="toc"><li><a href="#Foo" title="Sub Clause"><span class="secnum">1.1</span> Sub Clause</a></li></ol></li><li><a href="#sec-number" title="Explicit number"><span class="secnum">7</span> Explicit number</a><ol class="toc"><li><a href="#sec-number1" title="Automatic number inside explicit number"><span class="secnum">7.1</span> Automatic number inside explicit number</a></li><li><a href="#sec-number2" title="Nested explicit number"><span class="secnum">7.4</span> Nested explicit number</a></li><li><a href="#sec-number3" title="Automatic number after explicit number"><span class="secnum">7.5</span> Automatic number after explicit number</a></li><li><a href="#sec-number-nested" title="Multi-step explicit numbers"><span class="secnum">7.6</span> Multi-step explicit numbers</a><ol class="toc"><li><a href="#sec-number-multi-step" title="Multi-step explicit numbers"><span class="secnum">7.6.1.1</span> Multi-step explicit numbers</a></li><li><a href="#sec-number-multi-step-1" title="Automatic number after explicit number"><span class="secnum">7.6.1.2</span> Automatic number after explicit number</a><ol class="toc"><li><a href="#sec-number-multi-step-inner" title="Nested clause after explicit multi-step number"><span class="secnum">7.6.1.2.1</span> Nested clause after explicit multi-step number</a></li></ol></li><li><a href="#sec-number-multi-step-2" title="Increasing multi-step explicit numbers"><span class="secnum">7.6.2.1</span> Increasing multi-step explicit numbers</a></li></ol></li></ol></li><li><a href="#sec-annex" title="Annex"><span class="secnum">A</span> Annex</a><ol class="toc"><li><a href="#sec-annex2" title="Sub-annex"><span class="secnum">A.1</span> Sub-annex</a></li></ol></li></ol></div><emu-intro id="sec-intro">
   <h1>Intro</h1>
   <emu-intro id="sec-intro2">
     <h1>Sub Intro</h1>
@@ -51,7 +51,7 @@ <h1><span class="secnum">7.6.1.2.1</span> Nested clause after explicit multi-ste
       </emu-clause>
     </emu-clause>
 
-    <emu-clause id="sec-number-multi-step" number="2.1">
+    <emu-clause id="sec-number-multi-step-2" number="2.1">
       <h1><span class="secnum">7.6.2.1</span> Increasing multi-step explicit numbers</h1>
     </emu-clause>
   </emu-clause>

test/baselines/generated-reference/duplicate-ids.html

@@ -8,6 +8,7 @@
 </ul></div><div id="spec-container">
 <emu-clause id="sec-a">
   <h1><span class="secnum">1</span> A</h1>
+  <!-- EXPECT_WARNING { "ruleId": "duplicate-id", "message": "<emu-clause> has duplicate id \"sec-a\"" } -->
   <emu-clause id="sec-a">
     <h1><span class="secnum">1.1</span> Sub A</h1>
 
@@ -16,6 +17,7 @@ <h1><span class="secnum">1.1</span> Sub A</h1>
     </figure></emu-example>
   </emu-clause>
 </emu-clause>
+<!-- EXPECT_WARNING { "ruleId": "duplicate-id", "message": "<emu-clause> has duplicate id \"sec-a\"" } -->
 <emu-clause id="sec-a">
   <h1><span class="secnum">2</span> Section A: Extras</h1>
   <emu-table id="table-of-stuff" caption="A Table Of Stuff" informative=""><figure><figcaption>Table 1 (Informative): A Table Of Stuff</figcaption>
@@ -25,14 +27,17 @@ <h1><span class="secnum">2</span> Section A: Extras</h1>
     </tbody></table>
   </figure></emu-table>
 </emu-clause>
+<!-- EXPECT_WARNING { "ruleId": "duplicate-id", "message": "<emu-clause> has duplicate id \"sec-a\"" } -->
 <emu-clause id="sec-a">
   <h1><span class="secnum">3</span> Section A: Extras</h1>
+  <!-- EXPECT_WARNING { "ruleId": "duplicate-id", "message": "<emu-table> has duplicate id \"table-of-stuff\"" } -->
   <emu-table id="table-of-stuff" caption="A Table Of Stuff" informative=""><figure><figcaption>Table 2 (Informative): A Table Of Stuff</figcaption>
     <table>
       <tbody><tr><th>Column 1</th><th>Column 2</th></tr>
       <tr><td>Value</td><td>Value 2</td></tr>
     </tbody></table>
   </figure></emu-table>
+  <!-- EXPECT_WARNING { "ruleId": "duplicate-id", "message": "<emu-example> has duplicate id \"an-example\"" } -->
   <emu-example id="an-example" caption="An example"><figure><figcaption>Example (Informative): An example</figcaption>
     Multiple examples are numbered similar to notes
   </figure></emu-example>

test/baselines/generated-reference/figure.html

@@ -38,12 +38,12 @@
   </tbody></table>
 </figure></emu-table>
 
-<emu-figure id="figure-2" informative=""><figure><figcaption>Figure 3 (Informative): This is the caption</figcaption>
+<emu-figure id="figure-3" informative=""><figure><figcaption>Figure 3 (Informative): This is the caption</figcaption>
 
   this is a figure!
 </figure></emu-figure>
 
-<emu-table id="table-1"><figure><figcaption>Table 3: This is a table</figcaption>
+<emu-table id="table-3"><figure><figcaption>Table 3: This is a table</figcaption>
 
   <table>
     <thead>
@@ -56,7 +56,7 @@
   </tbody></table>
 </figure></emu-table>
 
-<emu-table id="table-2" informative=""><figure><figcaption>Table 4 (Informative): This is a <b>second</b> table</figcaption>
+<emu-table id="table-4" informative=""><figure><figcaption>Table 4 (Informative): This is a <b>second</b> table</figcaption>
 
   <table>
     <tbody><tr><th>Column 1</th><th>Column 2</th></tr>

test/baselines/generated-reference/namespaces.html

@@ -44,6 +44,8 @@ <h1><span class="secnum">1.1</span> Clause 1.1</h1>
 </emu-production>
 </emu-grammar>
 
+    <!-- FIXME: Shouldn't this be allowed by the preceding `namespace=clause`? -->
+    <!-- EXPECT_WARNING { "ruleId": "duplicate-definition", "message": "duplicate definition \"SomeAlg\"" } -->
     <emu-clause id="c111" aoid="SomeAlg">
       <h1><span class="secnum">1.1.1</span> SomeAlg</h1>
     </emu-clause>
@@ -63,10 +65,14 @@ <h1><span class="secnum">A</span> Annex</h1>
     </emu-rhs>
 </emu-production>
 </emu-grammar>
+  <!-- FIXME: Shouldn't this be allowed by the preceding `namespace=annex`? -->
+  <!-- EXPECT_WARNING { "ruleId": "duplicate-definition", "message": "duplicate definition \"SomeAlg\"" } -->
   <emu-annex id="annex11" aoid="SomeAlg">
     <h1><span class="secnum">A.1</span> SomeAlg</h1>
   </emu-annex>
 
+  <!-- FIXME: Shouldn't this be allowed by `namespace=annex2`? -->
+  <!-- EXPECT_WARNING { "ruleId": "duplicate-definition", "message": "duplicate definition \"SomeAlg\"" } -->
   <emu-annex id="annex12" aoid="SomeAlg" namespace="annex2">
     <h1><span class="secnum">A.2</span> Annex 1.2</h1>
     <p><emu-xref aoid="SomeAlg" id="_ref_3"><a href="#i1">SomeAlg</a></emu-xref> should link to #annex12. <emu-nt id="_ref_12"><a href="#prod-annex-Foo">Foo</a></emu-nt> should link to the production in #annex1.</p>
@@ -78,6 +84,7 @@ <h1><span class="secnum">A.2</span> Annex 1.2</h1>
 
 <emu-annex id="annex2" namespace="annex">
   <h1><span class="secnum">B</span> Annex 2</h1>
+  <!-- EXPECT_WARNING { "ruleId": "duplicate-definition", "message": "duplicate definition \"SomeAlg\"" } -->
   <emu-annex id="annex21" aoid="SomeAlg">
     <h1><span class="secnum">B.1</span> SomeAlg</h1>
   </emu-annex>

test/baselines/generated-reference/structured-headers.html

@@ -17,6 +17,7 @@ <h1><span class="secnum">1</span> ExampleAO ( param [ , param2 ] )</h1>
   <emu-alg><ol><li>Algorithm steps go here.</li></ol></emu-alg>
 </emu-clause>
 
+<!-- EXPECT_WARNING { "ruleId": "duplicate-id", "message": "<emu-clause> has duplicate id \"sec-exampleao\"" } -->
 <emu-clause id="sec-exampleao" type="abstract operation">
   <h1><span class="secnum">2</span> ExampleAO ( )</h1>
   <p>The abstract operation ExampleAO takes no arguments. It redefines an existing algorithm. It performs the following steps when called:</p>
@@ -87,9 +88,9 @@ <h1><span class="secnum">6.2</span> IsThat</h1>
   </emu-clause>
 </emu-clause>
 
-<emu-clause id="sec-example-return-type" type="abstract operation" aoid="ExampleAO3">
-  <h1><span class="secnum">7</span> ExampleAO3 ( param )</h1>
-  <p>The abstract operation <emu-xref aoid="ExampleAO3" id="_ref_0"><a href="#sec-exampleao3">ExampleAO3</a></emu-xref> takes argument param (an <emu-xref href="#integer"><a href="https://tc39.es/ecma262/#integer">integer</a></emu-xref>) and returns the return type. It performs the following steps when called:</p>
+<emu-clause id="sec-example-return-type" type="abstract operation" aoid="ExampleAO5">
+  <h1><span class="secnum">7</span> ExampleAO5 ( param )</h1>
+  <p>The abstract operation ExampleAO5 takes argument param (an <emu-xref href="#integer"><a href="https://tc39.es/ecma262/#integer">integer</a></emu-xref>) and returns the return type. It performs the following steps when called:</p>
   <emu-alg><ol><li>Algorithm steps go here.</li></ol></emu-alg>
 </emu-clause>
 

test/baselines/sources/clauses.html

@@ -48,7 +48,7 @@ <h1>Nested clause after explicit multi-step number</h1>
       </emu-clause>
     </emu-clause>
 
-    <emu-clause id="sec-number-multi-step" number="2.1">
+    <emu-clause id="sec-number-multi-step-2" number="2.1">
       <h1>Increasing multi-step explicit numbers</h1>
     </emu-clause>
   </emu-clause>