feat!: JSON doc generation (#533)

Supersedes #481
Part of cdklabs/construct-hub-webapp#714 and cdklabs/construct-hub#142

Allows rendering language specific documentation to a JSON object in addition to a markdown string. This allows the consumer more flexibility to render the structure of the API reference as needed without regenerating markdown documents.

This PR builds upon #481 by cleaning up the JSON schema (in `schema.ts`) and also changing the markdown rendering logic to be based on the JSON output -- so the markdown rendering does not depend on any `TranspiledXxx` classes or interfaces.

All headers now have anchor tags which default to the unique language-agnostic FQNs (e.g. `@aws-cdk/aws-s3.Bucket.parameter.scope`). Based on my research, all characters used in JSII FQNs (A-Za-z0-9_-/.@) are allowed as URL fragments, and are also allowed as id's in HTML5, so this is a reasonable default.

This PR also extends the notion of fully qualified names to all "entities" in the document (methods, properties, enum members, and even method parameters) denoted by the "id" fields within the JSON schema, so that any section of a document can be uniquely linked - not just types.  The `JsiiEntity` interface in `schema.ts` is intended to encapsulate any information that should be needed to render a link, including the package version it's from (this is necessary for Construct Hub to be able to link to the correct versions of packages).

Markdown generation can now be customized using three hooks:

- `anchorFormatter: (type: JsiiEntity) => string` - customize the IDs given to anchors tags that are placed next to every header in the document, so that any part of the API reference can be directly linked
- `linkFormatter: (type: JsiiEntity, metadata: AssemblyMetadataSchema) => string` - customize how links should be rendered when they appear in tables, descriptions, etc. -- e.g. to include logic to "link to another page"
- `typeFormatter?: (type: TypeSchema, linkFormatter) => string` - customize how composite types should be rendered when they include nested types - this allows functionality like the screenshot below:

<img width="912" alt="Screen Shot 2021-12-23 at 7 34 14 PM" src="https://user-images.githubusercontent.com/5008987/147302108-d8575415-f235-4ea8-91cb-db570f86274a.png">

See `markdown.test.ts` for an example of how the markdown rendering hooks would be used to format links for Construct Hub.

Examples:

@aws-cdk/aws-ecr Python API.json: https://gist.github.com/Chriscbr/731b315808faaf2a2161d686ce248368
@aws-cdk/aws-ecr Python API.md: https://gist.github.com/Chriscbr/d0833a9ecce258dead648a63253b8b0e

BREAKING CHANGE: `Documentation.render` is now named `Documentation.toMarkdown`. In addition, the options parameter for this method has changed. `TranspiledType` is no longer exported.

README.md

@@ -15,13 +15,20 @@ Will produce a file called `API.md` with the api reference for this module.
 As a library:
 
 ```ts
-import { Documentation } from 'jsii-docgen';
+import { Documentation, Language } from 'jsii-docgen';
 
-const docs = await Documentation.forLocalPackage('.', { language: 'ts' });
-const markdown = docs.render().render(); // returns a markdown string
+const docs = await Documentation.forProject('.');
+const markdown = await docs.toMarkdown({ language: Language.TYPESCRIPT }).render(); // returns a markdown string
+
+const json = await docs.toJson({ language: Language.TYPESCRIPT }).render(); // returns a JSON object
 ```
 
-Note that you can pass in either `ts` or `python` as the language, as opposed to the CLI, which only produces a TypeScript reference.
+Curreently jsii-docgen supports generating documentation in the following languages:
+
+- TypeScript (`typescript`)
+- Python (`python`)
+- Java (`java`)
+- C# (`csharp` or `dotnet`)
 
 ## Contributions
 

src/cli.ts

@@ -6,16 +6,20 @@ import { Documentation } from './index';
 export async function main() {
   const args = yargs
     .usage('Usage: $0')
-    .option('output', { type: 'string', alias: 'o', required: false, desc: 'Output filename (defaults to API.md)' })
+    .option('output', { type: 'string', alias: 'o', required: false, desc: 'Output filename (defaults to API.md if format is markdown, and API.json if format is JSON)' })
+    .option('format', { alias: 'f', default: 'md', choices: ['md', 'json'], desc: 'Output format, markdown or json' })
     .option('language', { alias: 'l', default: 'typescript', choices: Language.values().map(x => x.toString()), desc: 'Output language' })
     .example('$0', 'Generate documentation for the current module as a single file (auto-resolves node depedencies)')
     .argv;
 
   const language = Language.fromString(args.language);
   const docs = await Documentation.forProject(process.cwd());
-  const output = args.output ?? 'API.md';
-  const markdown = await docs.render({ readme: false, language });
-  fs.writeFileSync(output, markdown.render());
+  const options = { readme: false, language };
+  const fileSuffix = args.format === 'md' ? 'md' : 'json';
+  const output = args.output ?? `API.${fileSuffix}`;
+
+  const content = await (args.format === 'md' ? docs.toMarkdown(options) : docs.toJson(options));
+  fs.writeFileSync(output, content.render());
 }
 
 main().catch(e => {

src/docgen/render/json.ts

@@ -0,0 +1,10 @@
+/**
+ * Type-safe Json renderer.
+ */
+export class Json<T> {
+  constructor(public readonly content: T) {};
+
+  public render(replacer?: any, space?: string | number): string {
+    return JSON.stringify(this.content, replacer, space);
+  }
+}

src/docgen/render/markdown.ts → src/docgen/render/markdown-doc.ts

@@ -1,15 +1,4 @@
-import * as reflect from 'jsii-reflect';
-
-const sanitize = (input: string): string => {
-  return input
-    .toLowerCase()
-    .replace(/[^a-zA-Z0-9 ]/g, '')
-    .replace(/ /g, '-');
-};
-
-export const anchorForId = (id: string): string => {
-  return sanitize(id);
-};
+import { DocsSchema } from '../schema';
 
 /**
  * Options for defining a markdown header.
@@ -64,11 +53,11 @@ export interface MarkdownOptions {
 /**
  * Markdown element.
  */
-export class Markdown {
+export class MarkdownDocument {
   /**
    * An empty markdown element.
    */
-  public static readonly EMPTY = new Markdown();
+  public static readonly EMPTY = new MarkdownDocument();
 
   /**
    * Sanitize markdown reserved characters from external input.
@@ -90,15 +79,16 @@ export class Markdown {
   }
 
   public static pre(text: string): string {
-    return `\`${text}\``;
+    // using <code> instead of backticks since this allows links
+    return `<code>${text}</code>`;
   }
 
   public static italic(text: string) {
     return `*${text}*`;
   }
 
   private readonly _lines = new Array<string>();
-  private readonly _sections = new Array<Markdown>();
+  private readonly _sections = new Array<MarkdownDocument>();
 
   private readonly id?: string;
   private readonly header?: string;
@@ -109,25 +99,22 @@ export class Markdown {
   }
 
   /**
-   * Render a `jsii-reflect.Docs` element into the markdown.
+   * Render a docs element into the markdown.
    */
-  public docs(docs: reflect.Docs) {
+  public docs(docs: DocsSchema) {
     if (docs.summary) {
-      this.lines(Markdown.sanitize(docs.summary));
+      this.lines(MarkdownDocument.sanitize(docs.summary));
       this.lines('');
     }
     if (docs.remarks) {
-      this.lines(Markdown.sanitize(docs.remarks));
+      this.lines(MarkdownDocument.sanitize(docs.remarks));
       this.lines('');
     }
 
-    if (docs.docs.see) {
-      this.quote(docs.docs.see);
-    }
-
-    const customLink = docs.customTag('link');
-    if (customLink) {
-      this.quote(`[${customLink}](${customLink})`);
+    if (docs.links) {
+      for (const link of docs.links) {
+        this.quote(`[${link}](${link})`);
+      }
     }
   }
 
@@ -166,7 +153,7 @@ export class Markdown {
     this.lines('');
   }
 
-  public section(section: Markdown) {
+  public section(section: MarkdownDocument) {
     this._sections.push(section);
   }
 
@@ -180,20 +167,16 @@ export class Markdown {
 
     const content: string[] = [];
     if (this.header) {
-      const anchor = anchorForId(this.id ?? '');
       const heading = `${'#'.repeat(headerSize)} ${this.header}`;
 
-      // This is nasty, i'm aware.
-      // Its just an escape hatch so that produce working links by default, but also support producing the links that construct-hub currently relies on.
-      // This will be gone soon.
-      // Note though that cross links (i.e links dependencies will not work yet regardless)
+      // temporary hack to avoid breaking Construct Hub
       const headerSpan = !!process.env.HEADER_SPAN;
       if (headerSpan) {
         content.push(
-          `${heading} <span data-heading-title="${this.header}" data-heading-id="${anchor}"></span>`,
+          `${heading} <span data-heading-title="${this.options.header?.title}" data-heading-id="${this.id}"></span>`,
         );
       } else {
-        content.push(`${heading} <a name="${this.id}" id="${anchor}"></a>`);
+        content.push(`${heading} <a name="${this.options.header?.title}" id="${this.id}"></a>`);
       }
       content.push('');
     }