From 4b0da6923f61df57cbf641acc23a880458307f2b Mon Sep 17 00:00:00 2001
From: Angus Hollands <goosey15@gmail.com>
Date: Mon, 25 Mar 2024 13:32:01 +0000
Subject: [PATCH 1/9] feat: add csv-table directive

---
 package-lock.json                     |  6 ++
 packages/myst-directives/package.json |  1 +
 packages/myst-directives/src/table.ts | 94 ++++++++++++++++++++++++++-
 3 files changed, 100 insertions(+), 1 deletion(-)

diff --git a/package-lock.json b/package-lock.json
index 883b38b44..e1a95da83 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -15414,6 +15414,7 @@
       "license": "MIT",
       "dependencies": {
         "classnames": "^2.3.2",
+        "csv-parse": "^5.5.5",
         "js-yaml": "^4.1.0",
         "myst-common": "^1.2.0",
         "myst-spec-ext": "^1.2.0",
@@ -15422,6 +15423,11 @@
         "vfile": "^5.3.7"
       }
     },
+    "packages/myst-directives/node_modules/csv-parse": {
+      "version": "5.5.5",
+      "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.5.tgz",
+      "integrity": "sha512-erCk7tyU3yLWAhk6wvKxnyPtftuy/6Ak622gOO7BCJ05+TYffnPCJF905wmOQm+BpkX54OdAl8pveJwUdpnCXQ=="
+    },
     "packages/myst-execute": {
       "version": "0.0.5",
       "license": "MIT",
diff --git a/packages/myst-directives/package.json b/packages/myst-directives/package.json
index de00a65ae..47c9294e7 100644
--- a/packages/myst-directives/package.json
+++ b/packages/myst-directives/package.json
@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "classnames": "^2.3.2",
+    "csv-parse": "^5.5.5",
     "js-yaml": "^4.1.0",
     "myst-common": "^1.2.0",
     "myst-spec-ext": "^1.2.0",
diff --git a/packages/myst-directives/src/table.ts b/packages/myst-directives/src/table.ts
index 48cd1ee36..a49f59f25 100644
--- a/packages/myst-directives/src/table.ts
+++ b/packages/myst-directives/src/table.ts
@@ -1,6 +1,7 @@
-import type { DirectiveSpec, DirectiveData, GenericNode } from 'myst-common';
+import type { DirectiveSpec, DirectiveData, DirectiveContext, GenericNode } from 'myst-common';
 import { fileError, normalizeLabel, RuleId } from 'myst-common';
 import type { VFile } from 'vfile';
+import { parse } from 'csv-parse/sync';
 
 export const tableDirective: DirectiveSpec = {
   name: 'table',
@@ -162,3 +163,94 @@ export const listTableDirective: DirectiveSpec = {
     return [container];
   },
 };
+
+export const csvTableDirective: DirectiveSpec = {
+  name: 'csv-table',
+  arg: {
+    type: 'myst',
+  },
+  options: {
+    label: {
+      type: String,
+      alias: ['name'],
+    },
+    'header-rows': {
+      type: Number,
+      // nonnegative int
+    },
+    class: {
+      type: String,
+      // class_option: list of strings?
+      doc: `CSS classes to add to your table. Special classes include:
+
+- \`full-width\`: changes the table environment to cover two columns in LaTeX`,
+    },
+    align: {
+      type: String,
+      // choice(['left', 'center', 'right'])
+    },
+    delim: {
+      type: String,
+    },
+    escape: {
+      type: String,
+    },
+    keepspace: {
+      type: Boolean,
+    },
+    quote: {
+      type: String,
+    },
+  },
+  body: {
+    type: String,
+    required: true,
+  },
+  run(data: DirectiveData, vfile: VFile, ctx: DirectiveContext): GenericNode[] {
+    const delimiter = (data.options?.delimiter ?? ',') as string;
+    const records = parse(data.body as string, {
+      delimiter,
+      ltrim: !data.options?.keepspace,
+      escape: (data.options?.escape ?? delimiter) as string,
+      quote: (data.options?.quote ?? '"') as string,
+    });
+
+    const { label, identifier } = normalizeLabel(data.options?.label as string | undefined) || {};
+
+    let headerCount = (data.options?.['header-rows'] as number) || 0;
+    const rows = records.map((record: any, recordIndex: number) => {
+      const cells = record.map((cell: string) => {
+        const rawCells = ctx.parseMyst(cell, recordIndex);
+        if (!(rawCells.length === 1 && rawCells[0].type === 'paragraph')) {
+          throw new Error(`Expected a single paragraph node, encountered ${rawCells[0].type}`);
+        }
+        return {
+          type: 'tableCell',
+          header: headerCount > 0 ? true : undefined,
+          children: rawCells[0].children,
+        };
+      });
+      headerCount -= 1;
+      // Parsing produes multiple nodes
+      return {
+        type: 'tableRow',
+        children: cells,
+      };
+    });
+    const table = {
+      type: 'table',
+      align: data.options?.align,
+      children: rows,
+    };
+    const container = {
+      type: 'container',
+      kind: 'table',
+      identifier: identifier,
+      label: label,
+      class: data.options?.class,
+      children: [...(data.arg as GenericNode[]), table],
+    };
+
+    return [container];
+  },
+};

From 252838119490f883d6346963333adf67dde7e16f Mon Sep 17 00:00:00 2001
From: Angus Hollands <goosey15@gmail.com>
Date: Wed, 17 Apr 2024 17:03:58 +0100
Subject: [PATCH 2/9] feat: use CSV glossary

---
 packages/myst-directives/src/index.ts |  5 +-
 packages/myst-directives/src/table.ts | 98 +++++++++++++++++++--------
 2 files changed, 74 insertions(+), 29 deletions(-)

diff --git a/packages/myst-directives/src/index.ts b/packages/myst-directives/src/index.ts
index 39f99b95f..2a855bb84 100644
--- a/packages/myst-directives/src/index.ts
+++ b/packages/myst-directives/src/index.ts
@@ -7,7 +7,7 @@ import { figureDirective } from './figure.js';
 import { iframeDirective } from './iframe.js';
 import { imageDirective } from './image.js';
 import { includeDirective } from './include.js';
-import { tableDirective, listTableDirective } from './table.js';
+import { csvTableDirective, tableDirective, listTableDirective } from './table.js';
 import { asideDirective } from './aside.js';
 import { glossaryDirective } from './glossary.js';
 import { mathDirective } from './math.js';
@@ -21,6 +21,7 @@ import { rawDirective } from './raw.js';
 export const defaultDirectives = [
   admonitionDirective,
   bibliographyDirective,
+  csvTableDirective,
   codeDirective,
   codeCellDirective,
   dropdownDirective,
@@ -51,7 +52,7 @@ export { figureDirective } from './figure.js';
 export { iframeDirective } from './iframe.js';
 export { imageDirective } from './image.js';
 export { includeDirective } from './include.js';
-export { listTableDirective, tableDirective } from './table.js';
+export { csvTableDirective, listTableDirective, tableDirective } from './table.js';
 export { asideDirective } from './aside.js';
 export { mathDirective } from './math.js';
 export { mdastDirective } from './mdast.js';
diff --git a/packages/myst-directives/src/table.ts b/packages/myst-directives/src/table.ts
index a49f59f25..c5c61a7e5 100644
--- a/packages/myst-directives/src/table.ts
+++ b/packages/myst-directives/src/table.ts
@@ -1,4 +1,10 @@
-import type { DirectiveSpec, DirectiveData, DirectiveContext, GenericNode } from 'myst-common';
+import type {
+  DirectiveSpec,
+  DirectiveData,
+  DirectiveContext,
+  GenericNode,
+  GenericParent,
+} from 'myst-common';
 import { fileError, normalizeLabel, RuleId } from 'myst-common';
 import type { VFile } from 'vfile';
 import { parse } from 'csv-parse/sync';
@@ -164,6 +170,34 @@ export const listTableDirective: DirectiveSpec = {
   },
 };
 
+function parseCSV(
+  data: string,
+  opts: DirectiveData['options'] | undefined,
+  ctx: DirectiveContext,
+): GenericParent[][] {
+  const delimiter = (opts?.delimiter ?? ',') as string;
+  const records = parse(data, {
+    delimiter,
+    ltrim: !opts?.keepspace,
+    escape: (opts?.escape ?? delimiter) as string,
+    quote: (opts?.quote ?? '"') as string,
+  });
+
+  return records.map((record: any, recordIndex: number) => {
+    return record.map((cell: string) => {
+      const rawResult = ctx.parseMyst(cell, recordIndex);
+      if (rawResult.type !== 'root') {
+        throw new Error(`Expected a root element from parsing MyST: ${cell}`);
+      }
+      const { children: rawCells } = rawResult;
+      if (!(rawCells.length === 1 && rawCells[0].type === 'paragraph')) {
+        throw new Error(`Expected a single paragraph node, encountered ${rawCells[0].type}`);
+      }
+      return rawCells[0];
+    });
+  });
+}
+
 export const csvTableDirective: DirectiveSpec = {
   name: 'csv-table',
   arg: {
@@ -174,6 +208,11 @@ export const csvTableDirective: DirectiveSpec = {
       type: String,
       alias: ['name'],
     },
+    header: {
+      type: String,
+      // nonnegative int
+    },
+
     'header-rows': {
       type: Number,
       // nonnegative int
@@ -207,36 +246,41 @@ export const csvTableDirective: DirectiveSpec = {
     required: true,
   },
   run(data: DirectiveData, vfile: VFile, ctx: DirectiveContext): GenericNode[] {
-    const delimiter = (data.options?.delimiter ?? ',') as string;
-    const records = parse(data.body as string, {
-      delimiter,
-      ltrim: !data.options?.keepspace,
-      escape: (data.options?.escape ?? delimiter) as string,
-      quote: (data.options?.quote ?? '"') as string,
-    });
-
     const { label, identifier } = normalizeLabel(data.options?.label as string | undefined) || {};
 
+    const rows: GenericParent[] = [];
+
+    if (data.options?.header !== undefined) {
+      const headerCells = parseCSV(data.options.header as string, data.options, ctx);
+      rows.push(
+        ...headerCells.map((parsedRow) => ({
+          type: 'tableRow',
+          children: parsedRow.map((parsedCell) => ({
+            type: 'tableCell',
+            header: true,
+            children: parsedCell.children,
+          })),
+        })),
+      );
+    }
+    const cells = parseCSV(data.body as string, data.options, ctx);
+
     let headerCount = (data.options?.['header-rows'] as number) || 0;
-    const rows = records.map((record: any, recordIndex: number) => {
-      const cells = record.map((cell: string) => {
-        const rawCells = ctx.parseMyst(cell, recordIndex);
-        if (!(rawCells.length === 1 && rawCells[0].type === 'paragraph')) {
-          throw new Error(`Expected a single paragraph node, encountered ${rawCells[0].type}`);
-        }
-        return {
-          type: 'tableCell',
-          header: headerCount > 0 ? true : undefined,
-          children: rawCells[0].children,
+    rows.push(
+      ...cells.map((parsedRow) => {
+        const row = {
+          type: 'tableRow',
+          children: parsedRow.map((parsedCell) => ({
+            type: 'tableCell',
+            header: headerCount > 0 ? true : undefined,
+            children: parsedCell.children,
+          })),
         };
-      });
-      headerCount -= 1;
-      // Parsing produes multiple nodes
-      return {
-        type: 'tableRow',
-        children: cells,
-      };
-    });
+
+        headerCount -= 1;
+        return row;
+      }),
+    );
     const table = {
       type: 'table',
       align: data.options?.align,

From 6b665af7d4054c1aa65648ce23b497005fa031b1 Mon Sep 17 00:00:00 2001
From: Angus Hollands <goosey15@gmail.com>
Date: Thu, 18 Apr 2024 11:30:44 +0100
Subject: [PATCH 3/9] chore: cleanups

---
 packages/myst-directives/src/table.ts | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/packages/myst-directives/src/table.ts b/packages/myst-directives/src/table.ts
index c5c61a7e5..59150e8a5 100644
--- a/packages/myst-directives/src/table.ts
+++ b/packages/myst-directives/src/table.ts
@@ -8,6 +8,7 @@ import type {
 import { fileError, normalizeLabel, RuleId } from 'myst-common';
 import type { VFile } from 'vfile';
 import { parse } from 'csv-parse/sync';
+import { select } from 'unist-util-select';
 
 export const tableDirective: DirectiveSpec = {
   name: 'table',
@@ -170,6 +171,13 @@ export const listTableDirective: DirectiveSpec = {
   },
 };
 
+/**
+ * Parse a CSV-table comprising of (inline) MyST Markdown
+ *
+ * @param data - CSV string
+ * @param opts - directive options
+ * @param ctx - directive evaluation context
+ */
 function parseCSV(
   data: string,
   opts: DirectiveData['options'] | undefined,
@@ -185,15 +193,13 @@ function parseCSV(
 
   return records.map((record: any, recordIndex: number) => {
     return record.map((cell: string) => {
-      const rawResult = ctx.parseMyst(cell, recordIndex);
-      if (rawResult.type !== 'root') {
-        throw new Error(`Expected a root element from parsing MyST: ${cell}`);
-      }
-      const { children: rawCells } = rawResult;
-      if (!(rawCells.length === 1 && rawCells[0].type === 'paragraph')) {
-        throw new Error(`Expected a single paragraph node, encountered ${rawCells[0].type}`);
+      const mdast = ctx.parseMyst(cell, recordIndex);
+      const paragraph = select('*:root > paragraph:only-child', mdast);
+
+      if (paragraph === undefined) {
+        throw new Error(`Expected a root element containing a paragraph, found: ${cell}`);
       }
-      return rawCells[0];
+      return paragraph;
     });
   });
 }

From 7584adea647b37d4c095363050a10cf8c8e0baf7 Mon Sep 17 00:00:00 2001
From: Angus Hollands <goosey15@gmail.com>
Date: Thu, 18 Apr 2024 14:09:25 +0100
Subject: [PATCH 4/9] chore: add changeset

---
 .changeset/silent-dancers-burn.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .changeset/silent-dancers-burn.md

diff --git a/.changeset/silent-dancers-burn.md b/.changeset/silent-dancers-burn.md
new file mode 100644
index 000000000..8a881601e
--- /dev/null
+++ b/.changeset/silent-dancers-burn.md
@@ -0,0 +1,5 @@
+---
+"myst-directives": minor
+---
+
+Add the csv-table directive

From b066d68d4fb0dbb28198f9a516fe550a8aa334f0 Mon Sep 17 00:00:00 2001
From: Angus Hollands <goosey15@gmail.com>
Date: Thu, 18 Apr 2024 14:30:19 +0100
Subject: [PATCH 5/9] docs: add small docs

---
 docs/directives.md |  3 +++
 docs/tables.md     | 21 +++++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/docs/directives.md b/docs/directives.md
index d22f459b0..e2438aa36 100644
--- a/docs/directives.md
+++ b/docs/directives.md
@@ -18,6 +18,9 @@ description: A full list of the directives included in MyST Markdown by default.
 :::{myst:directive} code-cell
 :::
 
+:::{myst:directive} csv-table
+:::
+
 :::{myst:directive} dropdown
 :::
 
diff --git a/docs/tables.md b/docs/tables.md
index 84f299974..619b564ac 100644
--- a/docs/tables.md
+++ b/docs/tables.md
@@ -36,6 +36,11 @@ You can use the {myst:directive}`table` directive to add a caption to a markdown
 :::
 ```
 
+```{note}
+You may have inline markdown in the table caption, however, if it includes backticks, you must use a [colon fence](#example-fence).
+```
+
+
 ## List Tables
 
 ````{myst}
@@ -52,11 +57,19 @@ You can use the {myst:directive}`table` directive to add a caption to a markdown
 ```
 ````
 
-```{note}
-You may have inline markdown in the table caption, however, if it includes backticks, you must use a [colon fence](#example-fence).
-```
+## CSV Tables
 
+````{myst}
+```{csv-table} This table title
+:header-rows: 1
+:label: example-table
+
+Training, Validation
+0,        5
+13720,    2744
+```
+````
 ## Notebook outputs as tables
 
 You can embed Jupyter Notebook outputs as tables.
-See [](reuse-jupyter-outputs.md) for more information.
\ No newline at end of file
+See [](reuse-jupyter-outputs.md) for more information.

From eb8e1727fa6e3e7931757772207426109cecc1ee Mon Sep 17 00:00:00 2001
From: Angus Hollands <goosey15@gmail.com>
Date: Thu, 18 Apr 2024 20:58:33 +0100
Subject: [PATCH 6/9] refactor: better errors

---
 packages/myst-directives/src/table.ts | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/packages/myst-directives/src/table.ts b/packages/myst-directives/src/table.ts
index 59150e8a5..ea03d7de8 100644
--- a/packages/myst-directives/src/table.ts
+++ b/packages/myst-directives/src/table.ts
@@ -257,7 +257,15 @@ export const csvTableDirective: DirectiveSpec = {
     const rows: GenericParent[] = [];
 
     if (data.options?.header !== undefined) {
-      const headerCells = parseCSV(data.options.header as string, data.options, ctx);
+      let headerCells: GenericParent[][] = [];
+      try {
+        headerCells = parseCSV(data.options.header as string, data.options, ctx);
+      } catch (error) {
+        fileError(vfile, 'csv-table directive header must be valid CSV-formatted MyST', {
+          node: select('mystDirectiveOption[name="tags"]', data.node) ?? data.node,
+          ruleId: RuleId.directiveOptionsCorrect,
+        });
+      }
       rows.push(
         ...headerCells.map((parsedRow) => ({
           type: 'tableRow',
@@ -269,11 +277,20 @@ export const csvTableDirective: DirectiveSpec = {
         })),
       );
     }
-    const cells = parseCSV(data.body as string, data.options, ctx);
+
+    let bodyCells: GenericParent[][] = [];
+    try {
+      bodyCells = parseCSV(data.body as string, data.options, ctx);
+    } catch (error) {
+      fileError(vfile, 'csv-table directive body must be valid CSV-formatted MyST', {
+        node: select('mystDirectiveBody', data.node) ?? data.node,
+        ruleId: RuleId.directiveBodyCorrect,
+      });
+    }
 
     let headerCount = (data.options?.['header-rows'] as number) || 0;
     rows.push(
-      ...cells.map((parsedRow) => {
+      ...bodyCells.map((parsedRow) => {
         const row = {
           type: 'tableRow',
           children: parsedRow.map((parsedCell) => ({
@@ -298,7 +315,7 @@ export const csvTableDirective: DirectiveSpec = {
       identifier: identifier,
       label: label,
       class: data.options?.class,
-      children: [...(data.arg as GenericNode[]), table],
+      children: [...((data.arg ?? []) as GenericNode[]), table],
     };
 
     return [container];

From f44229618852bdd80211bb5e8d6a6f3d654adccd Mon Sep 17 00:00:00 2001
From: Rowan Cockett <rowanc1@gmail.com>
Date: Fri, 19 Apr 2024 14:05:39 -0600
Subject: [PATCH 7/9] Add docs

---
 .changeset/tame-apricots-doubt.md     |  5 +++
 docs/myst.yml                         |  1 +
 docs/tables.md                        | 49 +++++++++++++++++++------
 packages/myst-directives/src/table.ts | 51 +++++++++++++++++++--------
 packages/myst-transforms/src/html.ts  |  8 +++++
 5 files changed, 89 insertions(+), 25 deletions(-)
 create mode 100644 .changeset/tame-apricots-doubt.md

diff --git a/.changeset/tame-apricots-doubt.md b/.changeset/tame-apricots-doubt.md
new file mode 100644
index 000000000..7bdf42440
--- /dev/null
+++ b/.changeset/tame-apricots-doubt.md
@@ -0,0 +1,5 @@
+---
+"myst-transforms": patch
+---
+
+Support span and div in html parser
diff --git a/docs/myst.yml b/docs/myst.yml
index f10f92593..f4eb9f5e1 100644
--- a/docs/myst.yml
+++ b/docs/myst.yml
@@ -44,6 +44,7 @@ project:
     JSON: JavaScript Object Notation
     TLA: Three Letter Acronym
     OA: Open Access
+    CSV: comma-separated values
   plugins:
     - directives.mjs
     - unsplash.mjs
diff --git a/docs/tables.md b/docs/tables.md
index 619b564ac..5856d6ba2 100644
--- a/docs/tables.md
+++ b/docs/tables.md
@@ -1,4 +1,6 @@
-# Tables
+---
+title: Tables
+---
 
 ## Github Flavoured
 
@@ -40,9 +42,11 @@ You can use the {myst:directive}`table` directive to add a caption to a markdown
 You may have inline markdown in the table caption, however, if it includes backticks, you must use a [colon fence](#example-fence).
 ```
 
-
 ## List Tables
 
+The {myst:directive}`list-table` directive is used to create a table from data in a uniform two-level bullet list.
+"Uniform" means that each sublist (second-level list) must contain the same number of list items.
+
 ````{myst}
 ```{list-table} This table title
 :header-rows: 1
@@ -59,16 +63,41 @@ You may have inline markdown in the table caption, however, if it includes backt
 
 ## CSV Tables
 
-````{myst}
-```{csv-table} This table title
-:header-rows: 1
-:label: example-table
+The {myst:directive}`csv-table` directive is used to create a table from comma-separated values (CSV) data.
+Block markup and inline markup within cells is supported. Line ends are recognized within quoted cells.
+
+```{csv-table} Frozen Delights!
+:header: "Treat", "Quantity", "Description"
 
-Training, Validation
-0,        5
-13720,    2744
+"Albatross", 2.99, "On a stick!"
+"Crunchy Frog", 1.49, "If we took the bones out
+it wouldn't be crunchy, now would it?"
+"Gannet Ripple", 1.99, "On a stick!"
 ```
-````
+
+## <span style="background: -webkit-linear-gradient(20deg, #09009f, #E743D9); -webkit-background-clip: text; -webkit-text-fill-color: transparent;">Complex Tables with Style</span>
+
+It is also possible to write tables in raw HTML with `rowspan` and `colspan`, as well as for example:
+
+```{myst}
+:::{table} Area Comparisons (written in fancy HTML)
+:label: tbl:areas-html
+
+<table>
+<tr><th rowspan="2">Projection</th><th colspan="3" align="center">Area in square miles</th></tr>
+<tr><th align="right">Large Horizontal Area</th><th align="right" style="background: -webkit-linear-gradient(20deg, #09009f, #E743D9); -webkit-background-clip: text; -webkit-text-fill-color: transparent;">Large Vertical Area</th><th align="right">Smaller Square Area<th></tr>
+<tr><td>Albers Equal Area</td><td align="right">7,498.7</td><td align="right">10,847.3</td><td align="right">35.8</td></tr>
+<tr><td>Web Mercator</td><td align="right">13,410.0</td><td align="right">18,271.4</td><td align="right">63.0</td></tr>
+<tr><td>Difference</td><td align="right" style="background-color: red;color: white">5,911.3</td><td align="right">7,424.1</td><td align="right">27.2</td></tr>
+<tr><td><bold>Percent Difference</bold></td><td align="right" style="background-color: green;color: white">44%</td><td align="right">41%</td><td align="right">43%</td></tr>
+</table>
+:::
+```
+
+:::{note} Styles are Only for HTML
+CSS styles are currently only used for HTML outputs and are not carried through to all export targets (e.g. LaTeX) and are primarily used for web.
+:::
+
 ## Notebook outputs as tables
 
 You can embed Jupyter Notebook outputs as tables.
diff --git a/packages/myst-directives/src/table.ts b/packages/myst-directives/src/table.ts
index ea03d7de8..18ce8fb86 100644
--- a/packages/myst-directives/src/table.ts
+++ b/packages/myst-directives/src/table.ts
@@ -14,6 +14,7 @@ export const tableDirective: DirectiveSpec = {
   name: 'table',
   arg: {
     type: 'myst',
+    doc: 'An optional table caption',
   },
   options: {
     label: {
@@ -62,6 +63,7 @@ export const listTableDirective: DirectiveSpec = {
   name: 'list-table',
   arg: {
     type: 'myst',
+    doc: 'An optional table caption',
   },
   options: {
     label: {
@@ -171,6 +173,12 @@ export const listTableDirective: DirectiveSpec = {
   },
 };
 
+type ParseCsvOptions = {
+  delim?: 'tab' | 'space' | string;
+  keepspace?: boolean;
+  quote?: string;
+  escape: string;
+};
 /**
  * Parse a CSV-table comprising of (inline) MyST Markdown
  *
@@ -178,17 +186,13 @@ export const listTableDirective: DirectiveSpec = {
  * @param opts - directive options
  * @param ctx - directive evaluation context
  */
-function parseCSV(
-  data: string,
-  opts: DirectiveData['options'] | undefined,
-  ctx: DirectiveContext,
-): GenericParent[][] {
-  const delimiter = (opts?.delimiter ?? ',') as string;
+function parseCSV(data: string, ctx: DirectiveContext, opts?: ParseCsvOptions): GenericParent[][] {
+  const delimiter = opts?.delim ?? ',';
   const records = parse(data, {
-    delimiter,
+    delimiter: delimiter === 'tab' ? '\t' : delimiter === 'space' ? ' ' : delimiter,
     ltrim: !opts?.keepspace,
-    escape: (opts?.escape ?? delimiter) as string,
-    quote: (opts?.quote ?? '"') as string,
+    escape: opts?.escape ?? '"',
+    quote: opts?.quote ?? '"',
   });
 
   return records.map((record: any, recordIndex: number) => {
@@ -204,24 +208,36 @@ function parseCSV(
   });
 }
 
+// Documentation is from Docutils
+// License is public domain: https://docutils.sourceforge.io/COPYING.html
 export const csvTableDirective: DirectiveSpec = {
   name: 'csv-table',
+  doc: 'The "csv-table" directive is used to create a table from CSV (comma-separated values) data.',
   arg: {
     type: 'myst',
+    doc: 'An optional table caption',
   },
   options: {
     label: {
       type: String,
       alias: ['name'],
     },
+    // file: {
+    //   type: String,
+    //   doc: 'The local filesystem path to a CSV data file.',
+    //   alias: ['url'],
+    //  Add this to the description for the directive:
+    //  The data may be internal (an integral part of the document) or external (a separate file).
+    // },
     header: {
       type: String,
       // nonnegative int
+      doc: 'Supplemental data for the table header, added independently of and before any header-rows from the main CSV data. Must use the same CSV format as the main CSV data.',
     },
-
     'header-rows': {
       type: Number,
       // nonnegative int
+      doc: 'The number of rows of CSV data to use in the table header. Defaults to 0.',
     },
     class: {
       type: String,
@@ -236,19 +252,24 @@ export const csvTableDirective: DirectiveSpec = {
     },
     delim: {
       type: String,
-    },
-    escape: {
-      type: String,
+      doc: 'The character used to separate data fields. The special values "tab" and "space" are converted to the respective whitespace characters. Defaults to "," (comma)',
     },
     keepspace: {
       type: Boolean,
+      doc: 'Treat whitespace immediately following the delimiter as significant. The default is to ignore such whitespace.',
     },
     quote: {
       type: String,
+      doc: 'The character used to quote fields containing special characters, such as the delimiter, quotes, or new-line characters. Must be a single character, defaults to `"` (a double quote)\\\nFor example, `First cell, "These commas, for example, are escaped", Next cell`',
+    },
+    escape: {
+      type: String,
+      doc: 'A character used to escape the delimiter or quote characters from the CSV parser. Must be a single character, defaults to `"` (a double quote) default is a double quote\\\nFor example, `First cell, "These quotes"", for example, are escaped", Next cell`',
     },
   },
   body: {
     type: String,
+    doc: 'The CSV content',
     required: true,
   },
   run(data: DirectiveData, vfile: VFile, ctx: DirectiveContext): GenericNode[] {
@@ -259,7 +280,7 @@ export const csvTableDirective: DirectiveSpec = {
     if (data.options?.header !== undefined) {
       let headerCells: GenericParent[][] = [];
       try {
-        headerCells = parseCSV(data.options.header as string, data.options, ctx);
+        headerCells = parseCSV(data.options.header as string, ctx, data.options as ParseCsvOptions);
       } catch (error) {
         fileError(vfile, 'csv-table directive header must be valid CSV-formatted MyST', {
           node: select('mystDirectiveOption[name="tags"]', data.node) ?? data.node,
@@ -280,7 +301,7 @@ export const csvTableDirective: DirectiveSpec = {
 
     let bodyCells: GenericParent[][] = [];
     try {
-      bodyCells = parseCSV(data.body as string, data.options, ctx);
+      bodyCells = parseCSV(data.body as string, ctx, data.options as ParseCsvOptions);
     } catch (error) {
       fileError(vfile, 'csv-table directive body must be valid CSV-formatted MyST', {
         node: select('mystDirectiveBody', data.node) ?? data.node,
diff --git a/packages/myst-transforms/src/html.ts b/packages/myst-transforms/src/html.ts
index c708844ec..39134b284 100644
--- a/packages/myst-transforms/src/html.ts
+++ b/packages/myst-transforms/src/html.ts
@@ -95,6 +95,14 @@ const defaultHtmlToMdastOptions: Record<keyof HtmlTransformOptions, any> = {
     _brKeep(h: H, node: any) {
       return h(node, '_break');
     },
+    span(h: H, node: any) {
+      const attrs = addClassAndIdentifier(node);
+      return h(node, 'span', attrs, all(h, node));
+    },
+    div(h: H, node: any) {
+      const attrs = addClassAndIdentifier(node);
+      return h(node, 'div', attrs, all(h, node));
+    },
     a(h: H, node: any) {
       const attrs = addClassAndIdentifier(node);
       attrs.url = String(node.properties.href || '');

From 782e0141ad4e57a1603b5bbff9190a7b2747b3d1 Mon Sep 17 00:00:00 2001
From: Rowan Cockett <rowanc1@gmail.com>
Date: Fri, 19 Apr 2024 14:12:52 -0600
Subject: [PATCH 8/9] Fix tests

---
 packages/myst-transforms/tests/html.yml | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/packages/myst-transforms/tests/html.yml b/packages/myst-transforms/tests/html.yml
index 8cd2d1c2c..74de04299 100644
--- a/packages/myst-transforms/tests/html.yml
+++ b/packages/myst-transforms/tests/html.yml
@@ -8,25 +8,31 @@ cases:
     after:
       type: root
       children:
-        - type: paragraph
+        - type: div
           children:
-            - type: text
-              value: '*some text*'
+            - type: paragraph
+              children:
+                - type: text
+                  value: '*some text*'
   - title: style
     before:
       type: root
       children:
         - type: html
-          value: <div><p><em>some text</em></p></div>
+          value: '<div style="background: red"><p><em>some text</em></p></div>'
     after:
       type: root
       children:
-        - type: paragraph
+        - type: div
+          style:
+            background: red
           children:
-            - type: emphasis
+            - type: paragraph
               children:
-                - type: text
-                  value: some text
+                - type: emphasis
+                  children:
+                    - type: text
+                      value: some text
   - title: image
     before:
       type: root

From d517f77e6df5937d24542711fcbe4524734166b0 Mon Sep 17 00:00:00 2001
From: Rowan Cockett <rowanc1@gmail.com>
Date: Fri, 19 Apr 2024 14:14:44 -0600
Subject: [PATCH 9/9] Change node error to look to the correct option

---
 packages/myst-directives/src/table.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/myst-directives/src/table.ts b/packages/myst-directives/src/table.ts
index 18ce8fb86..5f46f66ae 100644
--- a/packages/myst-directives/src/table.ts
+++ b/packages/myst-directives/src/table.ts
@@ -283,7 +283,7 @@ export const csvTableDirective: DirectiveSpec = {
         headerCells = parseCSV(data.options.header as string, ctx, data.options as ParseCsvOptions);
       } catch (error) {
         fileError(vfile, 'csv-table directive header must be valid CSV-formatted MyST', {
-          node: select('mystDirectiveOption[name="tags"]', data.node) ?? data.node,
+          node: select('mystDirectiveOption[name="header"]', data.node) ?? data.node,
           ruleId: RuleId.directiveOptionsCorrect,
         });
       }