👩‍🎓 JATS citation / affiliation fixes (#783)

* 👩‍🎓 Process parts along with content during jats render
* 👩‍🎓 Only render used, ordered references in jats
* 🤯 Return mutable copies from redux selectors
* 🔧 Prevent empty ref-list elements in jats

.changeset/gentle-socks-leave.md

@@ -0,0 +1,5 @@
+---
+'myst-to-jats': patch
+---
+
+Process parts along with content during jats render

.changeset/serious-insects-wash.md

@@ -0,0 +1,5 @@
+---
+'myst-to-jats': patch
+---
+
+Only render used, ordered references in jats

.changeset/violet-flies-reflect.md

@@ -0,0 +1,5 @@
+---
+'myst-cli': patch
+---
+
+Return mutable copies from redux selectors

packages/myst-cli/src/store/selectors.ts

@@ -4,16 +4,21 @@ import type { LocalProject, LocalProjectPage } from '../project/types.js';
 import type { RootState } from './reducers.js';
 import type { BuildWarning, ExternalLinkResult } from './types.js';
 
+function mutableCopy(obj?: Record<string, any>) {
+  if (!obj) return;
+  return JSON.parse(JSON.stringify(obj));
+}
+
 export function selectLocalProject(state: RootState, path: string): LocalProject | undefined {
-  return state.local.projects[resolve(path)];
+  return mutableCopy(state.local.projects[resolve(path)]);
 }
 
 export function selectAffiliation(state: RootState, id: string): string | undefined {
   return state.local.affiliations[id];
 }
 
 export function selectLocalSiteConfig(state: RootState, path: string): SiteConfig | undefined {
-  return state.local.config.sites[resolve(path)];
+  return mutableCopy(state.local.config.sites[resolve(path)]);
 }
 
 export function selectCurrentSiteConfig(state: RootState): SiteConfig | undefined {
@@ -24,11 +29,11 @@ export function selectCurrentSiteConfig(state: RootState): SiteConfig | undefine
   return { ...config, projects: [{ path }] };
 }
 
-export function selectCurrentSitePath(state: RootState) {
+export function selectCurrentSitePath(state: RootState): string | undefined {
   return state.local.config.currentSitePath;
 }
 
-export function selectCurrentSiteFile(state: RootState) {
+export function selectCurrentSiteFile(state: RootState): string | undefined {
   if (!state.local.config.currentSitePath) return undefined;
   return state.local.config.filenames[resolve(state.local.config.currentSitePath)];
 }
@@ -37,19 +42,19 @@ export function selectLocalProjectConfig(
   state: RootState,
   path: string,
 ): ProjectConfig | undefined {
-  return state.local.config.projects[resolve(path)];
+  return mutableCopy(state.local.config.projects[resolve(path)]);
 }
 
-export function selectCurrentProjectConfig(state: RootState) {
+export function selectCurrentProjectConfig(state: RootState): ProjectConfig | undefined {
   if (!state.local.config.currentProjectPath) return undefined;
-  return state.local.config.projects[resolve(state.local.config.currentProjectPath)];
+  return mutableCopy(state.local.config.projects[resolve(state.local.config.currentProjectPath)]);
 }
 
-export function selectCurrentProjectPath(state: RootState) {
+export function selectCurrentProjectPath(state: RootState): string | undefined {
   return state.local.config.currentProjectPath;
 }
 
-export function selectCurrentProjectFile(state: RootState) {
+export function selectCurrentProjectFile(state: RootState): string | undefined {
   if (!state.local.config.currentProjectPath) return undefined;
   return state.local.config.filenames[resolve(state.local.config.currentProjectPath)];
 }
@@ -61,7 +66,7 @@ export function selectLocalRawConfig(
   state: RootState,
   path: string,
 ): { raw: Record<string, any>; validated: Record<string, any> } | undefined {
-  return state.local.config.rawConfigs[resolve(path)];
+  return mutableCopy(state.local.config.rawConfigs[resolve(path)]);
 }
 
 export function selectReloadingState(state: RootState) {
@@ -100,7 +105,11 @@ export function selectFileInfo(state: RootState, path: string) {
   };
 }
 
-export function selectPageSlug(state: RootState, projectPath: string, path: string) {
+export function selectPageSlug(
+  state: RootState,
+  projectPath: string,
+  path: string,
+): string | undefined {
   const project = selectLocalProject(state, projectPath);
   if (!project) return undefined;
   if (path === project.file) return project.index;
@@ -111,11 +120,11 @@ export function selectPageSlug(state: RootState, projectPath: string, path: stri
 }
 
 export function selectLinkStatus(state: RootState, url: string): ExternalLinkResult | undefined {
-  return state.local.links[url];
+  return mutableCopy(state.local.links[url]);
 }
 
 export function selectFileWarnings(state: RootState, file: string): BuildWarning[] | undefined {
-  return state.local.warnings[file];
+  return mutableCopy(state.local.warnings[file]);
 }
 
 export function selectFileWarningsByRule(

packages/myst-to-jats/src/backmatter.ts

@@ -124,13 +124,21 @@ export function citeToJatsRef(state: IJatsSerializer, key: string, data: Citatio
   };
 }
 
-export function getRefList(state: IJatsSerializer, citations?: CitationRenderer): Element[] {
-  if (!citations || !Object.keys(citations).length) return [];
-  const elements = Object.keys(citations)
-    .sort()
-    .map((key) => {
+export function getRefList(
+  state: IJatsSerializer,
+  order?: string[],
+  citations?: CitationRenderer,
+): Element[] {
+  const elements = order
+    ?.map((key) => {
+      if (!citations?.[key]) {
+        state.warn(`unknown citation ${key}`);
+        return undefined;
+      }
       return citeToJatsRef(state, key, citations[key].cite);
-    });
+    })
+    .filter((e): e is Element => !!e);
+  if (!elements?.length) return [];
   return [{ type: 'element', name: 'ref-list', elements }];
 }
 
@@ -157,15 +165,17 @@ export function getBack(
     citations,
     footnotes,
     expressions,
+    referenceOrder,
   }: {
     citations?: CitationRenderer;
     footnotes?: Element[];
     expressions?: Element[];
+    referenceOrder?: string[];
   },
 ): Element[] {
   const elements = [
     ...(state.data.backSections ?? []),
-    ...getRefList(state, citations),
+    ...getRefList(state, referenceOrder, citations),
     ...getFootnotes(footnotes),
     ...getExpressions(expressions),
     ...(state.data.acknowledgments ? [state.data.acknowledgments] : []),

packages/myst-to-jats/src/index.ts

@@ -71,6 +71,7 @@ import type {
   DocumentOptions,
   JatsPart,
 } from './types.js';
+import { ACKNOWLEDGMENT_PARTS, ABSTRACT_PARTS } from './types.js';
 import {
   basicTransformations,
   referenceResolutionTransform,
@@ -565,6 +566,9 @@ const handlers: Handlers = {
       'ref-type': 'bibr',
       rid: label,
     };
+    if (!state.referenceOrder.includes(label)) {
+      state.referenceOrder.push(label);
+    }
     state.renderInline(node, 'xref', attrs);
   },
   footnoteReference(node, state) {
@@ -714,7 +718,7 @@ function renderPart(vfile: VFile, mdast: GenericParent, part: string | string[],
   });
   if (!partMdast) return undefined;
   const serializer = new JatsSerializer(vfile, partMdast as Root, opts);
-  return serializer.render().elements();
+  return serializer.render(true).elements();
 }
 
 function renderAbstract(vfile: VFile, mdast: GenericParent, def: JatsPart, opts?: Options) {
@@ -731,7 +735,7 @@ function renderAbstract(vfile: VFile, mdast: GenericParent, def: JatsPart, opts?
 }
 
 function renderAcknowledgments(vfile: VFile, mdast: GenericParent, opts?: Options) {
-  const elements = renderPart(vfile, mdast, ['acknowledgments', 'acknowledgements'], opts);
+  const elements = renderPart(vfile, mdast, ACKNOWLEDGMENT_PARTS, opts);
   if (!elements) return undefined;
   const acknowledgments: Element = { type: 'element', name: 'ack', elements };
   return acknowledgments;
@@ -758,6 +762,8 @@ class JatsSerializer implements IJatsSerializer {
   stack: Element[];
   footnotes: Element[];
   expressions: Element[];
+  referenceOrder: string[];
+  opts: Options;
 
   constructor(file: VFile, mdast: Root, opts?: Options) {
     this.file = file;
@@ -768,23 +774,31 @@ class JatsSerializer implements IJatsSerializer {
     this.stack = [{ type: 'element', elements: [] }];
     this.footnotes = [];
     this.expressions = [];
+    this.referenceOrder = [];
     this.handlers = opts?.handlers ?? handlers;
     this.mdast = copyNode(mdast);
-    if (opts?.extractAbstract) {
-      const abstractParts = opts.abstractParts ?? [{ part: 'abstract' }];
-      this.data.abstracts = abstractParts
-        .map((def) => renderAbstract(this.file, this.mdast, def, opts))
-        .filter((e) => !!e) as Element[];
-    }
-    this.data.acknowledgments = renderAcknowledgments(this.file, this.mdast, opts);
-    const backSections = opts?.backSections ?? [];
-    this.data.backSections = backSections
-      .map((def) => renderBackSection(this.file, this.mdast, def, opts))
-      .filter((e) => !!e) as Element[];
+    this.opts = opts ?? {};
     basicTransformations(this.mdast as any, opts ?? {});
   }
 
-  render() {
+  render(ignoreParts?: boolean) {
+    if (!ignoreParts) {
+      if (this.opts?.extractAbstract) {
+        const abstractParts =
+          this.opts.abstractParts ??
+          ABSTRACT_PARTS.map((part) => {
+            return { part };
+          });
+        this.data.abstracts = abstractParts
+          .map((def) => renderAbstract(this.file, this.mdast, def, this.opts))
+          .filter((e) => !!e) as Element[];
+      }
+      this.data.acknowledgments = renderAcknowledgments(this.file, this.mdast, this.opts);
+      const backSections = this.opts?.backSections ?? [];
+      this.data.backSections = backSections
+        .map((def) => renderBackSection(this.file, this.mdast, def, this.opts))
+        .filter((e) => !!e) as Element[];
+    }
     this.renderChildren(this.mdast);
     while (this.stack.length > 1) this.closeNode();
     return this;
@@ -794,7 +808,7 @@ class JatsSerializer implements IJatsSerializer {
     return this.stack[this.stack.length - 1];
   }
 
-  warn(message: string, node: GenericNode, source?: string, opts?: MessageInfo) {
+  warn(message: string, node?: GenericNode, source?: string, opts?: MessageInfo) {
     fileError(this.file, message, {
       ...opts,
       node,
@@ -803,7 +817,7 @@ class JatsSerializer implements IJatsSerializer {
     });
   }
 
-  error(message: string, node: GenericNode, source?: string, opts?: MessageInfo) {
+  error(message: string, node?: GenericNode, source?: string, opts?: MessageInfo) {
     fileError(this.file, message, {
       ...opts,
       node,
@@ -951,6 +965,7 @@ export class JatsDocument {
         citations: this.content.citations,
         footnotes: articleState.footnotes,
         expressions: articleState.expressions,
+        referenceOrder: articleState.referenceOrder,
       }),
       ...subArticleStates.map((state, ind) => {
         return this.subArticle(state, subArticles[ind], ind === 0 && isNotebookArticleRep);
@@ -1021,6 +1036,7 @@ export class JatsDocument {
         citations: content.citations,
         footnotes: subArticleState.footnotes,
         expressions: subArticleState.expressions,
+        referenceOrder: subArticleState.referenceOrder,
       }),
     ];
     const attributes: Record<string, any> = {};

packages/myst-to-jats/src/transforms/blocks.ts

@@ -1,28 +1,52 @@
 import type { Plugin } from 'unified';
 import type { Block } from 'myst-spec-ext';
-import type { GenericParent } from 'myst-common';
+import { selectBlockParts, type GenericParent } from 'myst-common';
 import { remove } from 'unist-util-remove';
 import { selectAll } from 'unist-util-select';
+import { ABSTRACT_PARTS, ACKNOWLEDGMENT_PARTS, type Options } from '../types.js';
+
+function blocksToKeep(tree: GenericParent, opts: Options) {
+  const keepParts: string[] = [...ACKNOWLEDGMENT_PARTS];
+  if (opts.extractAbstract) {
+    keepParts.push(...(opts.abstractParts?.map(({ part }) => part).flat() ?? ABSTRACT_PARTS));
+  }
+  if (opts.backSections) {
+    keepParts.push(...opts.backSections.map(({ part }) => part).flat());
+  }
+  return selectBlockParts(tree, keepParts);
+}
 
 /**
  * This transform does the following:
- * - Removes hidden or removed blocks from the tree
+ * - Removes hidden or removed blocks from the tree except those that are consumed as front/backmatter parts
  * - Removes hidden or removed outputs from the tree
  */
-export function blockTransform(tree: GenericParent) {
+export function blockTransform(tree: GenericParent, opts: Options) {
+  // Collect blocks that will be used as parts to make sure they are not deleted
+  const doNotDelete = blocksToKeep(tree, opts);
   // This could also be an output, etc.
   (selectAll('[visibility=remove],[visibility=hide]', tree) as Block[]).forEach((node) => {
     if (node.visibility === 'remove' || node.visibility === 'hide') {
       (node as any).type = '__delete__';
     }
   });
+  // Blocks are converted to sections - avoid doing this for part blocks
+  doNotDelete.forEach((node) => {
+    node.type = 'block-part' as any;
+  });
   const removed = remove(tree, '__delete__');
   if (removed === null) {
     // remove is unhappy if all children are removed - this forces it through
     tree.children = [];
   }
 }
 
-export const blockPlugin: Plugin<[], GenericParent, GenericParent> = () => (tree) => {
-  blockTransform(tree);
+export const blockPlugin: Plugin<[Options], GenericParent, GenericParent> = (opts) => (tree) => {
+  blockTransform(tree, opts);
 };
+
+export function restoreBlockPartTypeTransform(tree: GenericParent) {
+  selectAll('block-part', tree).forEach((node) => {
+    node.type = 'block';
+  });
+}

packages/myst-to-jats/src/transforms/index.ts

@@ -4,7 +4,7 @@ import { definitionTransform } from './definitions.js';
 import { containerTransform } from './containers.js';
 import { tableTransform } from './tables.js';
 import { sectionTransform } from './sections.js';
-import { blockTransform } from './blocks.js';
+import { blockTransform, restoreBlockPartTypeTransform } from './blocks.js';
 import { citeGroupTransform } from './citations.js';
 import type { Options } from '../types.js';
 import type { GenericParent } from 'myst-common';
@@ -17,12 +17,13 @@ export { blockTransform, blockPlugin } from './blocks.js';
 export { referenceTargetTransform, referenceResolutionTransform } from './references.js';
 
 export function basicTransformations(tree: GenericParent, opts: Options) {
-  blockTransform(tree);
+  blockTransform(tree, opts);
   definitionTransform(tree);
   containerTransform(tree);
   tableTransform(tree);
   sectionTransform(tree, opts);
   citeGroupTransform(tree);
+  restoreBlockPartTypeTransform(tree);
 }
 
 export const basicTransformationsPlugin: Plugin<[Options], GenericParent, GenericParent> =

packages/myst-to-jats/src/types.ts

@@ -22,6 +22,10 @@ export type MathPlugins = Required<PageFrontmatter>['math'];
 
 export type JatsPart = { part: string | string[]; type?: string; title?: string };
 
+export const ACKNOWLEDGMENT_PARTS = ['acknowledgments', 'acknowledgements'];
+
+export const ABSTRACT_PARTS = ['abstract'];
+
 export type Options = {
   handlers?: Record<string, Handler>;
   isNotebookArticleRep?: boolean;
@@ -61,7 +65,8 @@ export interface IJatsSerializer<D extends Record<string, any> = StateData> {
   stack: Element[];
   footnotes: Element[];
   expressions: Element[];
-  render: () => IJatsSerializer;
+  referenceOrder: string[];
+  render: (ignoreParts?: boolean) => IJatsSerializer;
   text: (value?: string) => void;
   renderChildren: (node: any) => void;
   renderInline: (node: GenericNode, name: string, attributes?: Attributes) => void;
@@ -70,6 +75,6 @@ export interface IJatsSerializer<D extends Record<string, any> = StateData> {
   openNode: (name: string, attributes?: Attributes) => void;
   closeNode: () => void;
   elements: () => Element[];
-  warn: (message: string, node: GenericNode, source?: string, opts?: MessageInfo) => void;
-  error: (message: string, node: GenericNode, source?: string, opts?: MessageInfo) => void;
+  warn: (message: string, node?: GenericNode, source?: string, opts?: MessageInfo) => void;
+  error: (message: string, node?: GenericNode, source?: string, opts?: MessageInfo) => void;
 }