fix: Move page generators to page folder

src/cmd/build/index.ts

@@ -7,7 +7,7 @@ import OpenapiIncluder from '@diplodoc/openapi-extension/includer';
 
 import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants';
 import {argvValidator} from '../../validator';
-import {ArgvService, Includers} from '../../services';
+import {ArgvService, Includers, SearchService} from '../../services';
 import {
     initLinterWorkers,
     processAssets,
@@ -166,6 +166,7 @@ function builder<T>(argv: Argv<T>) {
             type: 'boolean',
             group: 'Build options:',
         })
+        .option('search', {})
         .check(argvValidator)
         .example('yfm -i ./input -o ./output', '')
         .demandOption(
@@ -190,6 +191,7 @@ async function handler(args: Arguments<any>) {
             input: tmpInputFolder,
             output: tmpOutputFolder,
         });
+        SearchService.init(args.output);
         Includers.init([OpenapiIncluder as any]);
 
         const {
@@ -236,6 +238,8 @@ async function handler(args: Arguments<any>) {
 
             await processChangelogs();
 
+            await SearchService.release();
+
             // Copy all generated files to user' output folder
             shell.cp(
                 '-r',

src/constants.ts

@@ -1,4 +1,5 @@
 import {resolve} from 'path';
+import {platform} from 'process';
 const os = require('os');
 const notes = require('@diplodoc/transform/lib/plugins/notes');
 const anchors = require('@diplodoc/transform/lib/plugins/anchors');
@@ -108,6 +109,8 @@ export const YFM_PLUGINS = [
     blockAnchor,
 ];
 
+export const CARRIAGE_RETURN = platform === Platforms.WINDOWS ? '\r\n' : '\n';
+
 export const PROCESSING_FINISHED = 'Processing finished:';
 export const LINTING_FINISHED = 'Linting finished:';
 export const GETTING_ALL_CONTRIBUTORS = 'Getting all contributors.';

src/utils/markup.ts → src/pages/document.ts

@@ -1,20 +1,14 @@
 import {join} from 'path';
-import {platform} from 'process';
-import {cloneDeepWith, flatMapDeep, isArray, isObject, isString} from 'lodash';
 
-import {BUNDLE_FOLDER, CUSTOM_STYLE, Platforms, RTL_LANGS} from '../constants';
-import {LeadingPage, Resources, SinglePageResult, TextItems, VarsMetadata} from '../models';
+import {BUNDLE_FOLDER, CARRIAGE_RETURN, CUSTOM_STYLE, RTL_LANGS} from '../constants';
+import {LeadingPage, Resources, TextItems, VarsMetadata} from '../models';
 import {ArgvService, PluginService} from '../services';
-import {preprocessPageHtmlForSinglePage} from './singlePage';
 
 import {DocInnerProps, DocPageData, render} from '@diplodoc/client/ssr';
 import manifest from '@diplodoc/client/manifest';
-import {isFileExists, resolveRelativePath} from '@diplodoc/transform/lib/utilsFS';
 
 import {escape} from 'html-escaper';
 
-export const carriageReturn = platform === Platforms.WINDOWS ? '\r\n' : '\n';
-
 export interface TitleMeta {
     title?: string;
 }
@@ -62,7 +56,7 @@ export function generateStaticMarkup(
                         height: 100vh;
                     }
                 </style>
-                ${manifest.css
+                ${manifest.app.css
                     .filter((file: string) => isRTL === file.includes('.rtl.css'))
                     .map((url: string) => join(deepBasePath, BUNDLE_FOLDER, url))
                     .map((src: string) => `<link type="text/css" rel="stylesheet" href="${src}" />`)
@@ -76,7 +70,7 @@ export function generateStaticMarkup(
                    window.STATIC_CONTENT = ${staticContent}
                    window.__DATA__ = ${JSON.stringify(props)};
                 </script>
-                ${manifest.js
+                ${manifest.app.js
                     .map((url: string) => join(deepBasePath, BUNDLE_FOLDER, url))
                     .map(
                         (src: string) =>
@@ -117,7 +111,7 @@ function getMetadata(metadata: VarsMetadata | undefined, restMeta: LeadingPage['
         }, '');
 
         if (args.length) {
-            result += `<meta ${args} />` + carriageReturn;
+            result += `<meta ${args} />` + CARRIAGE_RETURN;
         }
     };
 
@@ -155,71 +149,3 @@ function getResources({style, script}: Resources) {
 
     return resourcesTags.join('\n');
 }
-
-export function joinSinglePageResults(
-    singlePageResults: SinglePageResult[],
-    root: string,
-    tocDir: string,
-): string {
-    const delimeter = `<hr class="yfm-page__delimeter">`;
-    return singlePageResults
-        .filter(({content}) => content)
-        .map(({content, path, title}) =>
-            preprocessPageHtmlForSinglePage(content, {root, path, tocDir, title}),
-        )
-        .join(delimeter);
-}
-
-export function replaceDoubleToSingleQuotes(str: string): string {
-    return str.replace(/"/g, "'");
-}
-
-export function findAllValuesByKeys(obj, keysToFind: string[]) {
-    return flatMapDeep(obj, (value: string | string[], key: string) => {
-        if (
-            keysToFind?.includes(key) &&
-            (isString(value) || (isArray(value) && value.every(isString)))
-        ) {
-            return [value];
-        }
-
-        if (isObject(value)) {
-            return findAllValuesByKeys(value, keysToFind);
-        }
-
-        return [];
-    });
-}
-
-export function modifyValuesByKeys(
-    originalObj,
-    keysToFind: string[],
-    modifyFn: (value: string) => string,
-) {
-    function customizer(value, key) {
-        if (keysToFind?.includes(key) && isString(value)) {
-            return modifyFn(value);
-        }
-    }
-
-    // Clone the object deeply with a customizer function that modifies matching keys
-    return cloneDeepWith(originalObj, customizer);
-}
-
-export function getLinksWithContentExtersion(link: string) {
-    return new RegExp(/^\S.*\.(md|ya?ml|html)$/gm).test(link);
-}
-
-export function getLinksWithExtension(link: string) {
-    const oneLineWithExtension = new RegExp(
-        /^\S.*\.(md|html|yaml|svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm,
-    );
-
-    return oneLineWithExtension.test(link);
-}
-
-export function checkPathExists(path: string, parentFilePath: string) {
-    const includePath = resolveRelativePath(parentFilePath, path);
-
-    return isFileExists(includePath);
-}

src/pages/index.ts

@@ -0,0 +1,3 @@
+export {generateStaticMarkup} from './document'
+export {generateStaticRedirect} from './redirect'
+export {generateStaticSearch} from './search'

src/utils/redirect.ts → src/pages/redirect.ts

@@ -1,7 +1,7 @@
+import {join} from 'path';
 import {BUNDLE_FOLDER, Lang, RTL_LANGS} from '../constants';
 import {PluginService} from '../services';
 
-import {join} from 'path';
 import manifest from '@diplodoc/client/manifest';
 
 export function generateStaticRedirect(lang: Lang, link: string): string {
@@ -20,7 +20,7 @@ export function generateStaticRedirect(lang: Lang, link: string): string {
                         height: 100vh;
                     }
                 </style>
-                ${manifest.css
+                ${manifest.app.css
                     .filter((file: string) => isRTL === file.includes('.rtl.css'))
                     .map((url: string) => join(BUNDLE_FOLDER, url))
                     .map((src: string) => `<link type="text/css" rel="stylesheet" href="${src}" />`)

src/resolvers/md2html.ts

@@ -16,14 +16,14 @@ import {LeadingPage, ResolverOptions, YfmToc} from '../models';
 import {ArgvService, PluginService, TocService} from '../services';
 import {getAssetsPublicPath, getAssetsRootPath, getVCSMetadata} from '../services/metadata';
 import {
-    generateStaticMarkup,
     getLinksWithContentExtersion,
     getVarsPerFile,
     getVarsPerRelativeFile,
     logger,
     modifyValuesByKeys,
     transformToc,
 } from '../utils';
+import {generateStaticMarkup} from '../pages';
 
 export interface FileTransformOptions {
     path: string;

src/services/metadata/utils.ts

@@ -1,6 +1,6 @@
-import {carriageReturn} from '../../utils';
+import {CARRIAGE_RETURN} from '~/constants';
 
 // IMO, we should just always apply this at the end of the whole processing pipeline,
 // not when dumping meta/front matter
 export const normalizeLineEndings = (input: string): string =>
-    input.replace(/\r?\n/g, carriageReturn);
+    input.replace(/\r?\n/g, CARRIAGE_RETURN);

src/steps/processPages.ts

@@ -24,15 +24,11 @@ import {
 } from '../models';
 import {resolveMd2HTML, resolveMd2Md} from '../resolvers';
 import {ArgvService, LeadingService, PluginService, TocService} from '../services';
-import {
-    generateStaticMarkup,
-    joinSinglePageResults,
-    logger,
-    transformTocForSinglePage,
-} from '../utils';
+import {generateStaticMarkup} from '~/pages/document';
+import {generateStaticRedirect} from '~/pages/redirect';
+import {joinSinglePageResults, logger, transformTocForSinglePage} from '../utils';
 import {getVCSConnector} from '../vcs-connector';
 import {VCSConnector} from '../vcs-connector/connector-models';
-import {generateStaticRedirect} from '../utils/redirect';
 
 const singlePageResults: Record<string, SinglePageResult[]> = {};
 const singlePagePaths: Record<string, Set<string>> = {};

src/utils/common.ts

@@ -0,0 +1,52 @@
+import {cloneDeepWith, flatMapDeep, isArray, isObject, isString} from 'lodash';
+import {isFileExists, resolveRelativePath} from '@diplodoc/transform/lib/utilsFS';
+
+export function findAllValuesByKeys(obj: object, keysToFind: string[]) {
+    return flatMapDeep(obj, (value: string | string[], key: string) => {
+        if (
+            keysToFind?.includes(key) &&
+            (isString(value) || (isArray(value) && value.every(isString)))
+        ) {
+            return [value];
+        }
+
+        if (isObject(value)) {
+            return findAllValuesByKeys(value, keysToFind);
+        }
+
+        return [];
+    });
+}
+
+export function modifyValuesByKeys(
+    originalObj: object,
+    keysToFind: string[],
+    modifyFn: (value: string) => string,
+) {
+    // Clone the object deeply with a customizer function that modifies matching keys
+    return cloneDeepWith(originalObj, function (value: unknown, key) {
+        if (keysToFind.includes(key as string) && isString(value)) {
+            return modifyFn(value);
+        }
+
+        return value;
+    });
+}
+
+export function getLinksWithContentExtersion(link: string) {
+    return new RegExp(/^\S.*\.(md|ya?ml|html)$/gm).test(link);
+}
+
+export function getLinksWithExtension(link: string) {
+    const oneLineWithExtension = new RegExp(
+        /^\S.*\.(md|html|yaml|svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm,
+    );
+
+    return oneLineWithExtension.test(link);
+}
+
+export function checkPathExists(path: string, parentFilePath: string) {
+    const includePath = resolveRelativePath(parentFilePath, path);
+
+    return isFileExists(includePath);
+}

src/utils/index.ts

@@ -1,5 +1,6 @@
+export * from './common';
 export * from './logger';
-export * from './markup';
+export * from './singlePage';
 export * from './url';
 export * from './path';
 export * from './toc';

src/utils/singlePage.ts

@@ -1,3 +1,5 @@
+import type {SinglePageResult} from '~/models';
+
 import HTMLElement from 'node-html-parser/dist/nodes/html';
 import {parse} from 'node-html-parser';
 import {relative, resolve, sep} from 'path';
@@ -232,3 +234,17 @@ export function preprocessPageHtmlForSinglePage(
 
     return root.toString();
 }
+
+export function joinSinglePageResults(
+    singlePageResults: SinglePageResult[],
+    root: string,
+    tocDir: string,
+): string {
+    const delimeter = `<hr class="yfm-page__delimeter">`;
+    return singlePageResults
+        .filter(({content}) => content)
+        .map(({content, path, title}) =>
+            preprocessPageHtmlForSinglePage(content, {root, path, tocDir, title}),
+        )
+        .join(delimeter);
+}

tests/units/services/authors.test.ts

@@ -1,4 +1,4 @@
-import * as units from 'utils/markup';
+import * as units from '../../utils';
 import {
     getAuthorDetails,
     updateAuthorMetadataStringByAuthorLogin,

tests/units/services/metadata.test.ts

@@ -1,6 +1,6 @@
 import {Contributor, Contributors, Metadata, MetaDataOptions} from 'models';
 import {getVCSMetadata} from 'services/metadata';
-import {replaceDoubleToSingleQuotes} from 'utils/markup';
+import {replaceDoubleToSingleQuotes} from '../../utils';
 import {VCSConnector} from 'vcs-connector/connector-models';
 
 const contributorFirst: Contributor = {

tests/utils.ts

@@ -66,3 +66,7 @@ export function getTestPaths(testRootPath: string): TestPaths {
         outputPath: resolve(join(testRootPath, 'output')),
     };
 }
+
+export function replaceDoubleToSingleQuotes(str: string): string {
+    return str.replace(/"/g, "'");
+}