From 06a9e5f6a404a8268ff57d356b9f767d210929b3 Mon Sep 17 00:00:00 2001 From: phukon Date: Sun, 7 Apr 2024 21:39:00 +0000 Subject: [PATCH 1/3] feat: Add verbose logging for build progress #1852 --- .../build-plugins/verbose-logs/fileCounter.js | 13 ++++++ .../build-plugins/verbose-logs/get-pages.js | 19 ++++++++ .../src/build-plugins/verbose-logs/index.js | 28 ++++++++++++ .../verbose-logs/progressCalculator.js | 10 +++++ .../verbose-logs/wait-creation.js | 29 ++++++++++++ .../src/build-plugins/verbose-logs/watcher.js | 45 +++++++++++++++++++ .../src/build-plugins/vite.js | 3 +- 7 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/fileCounter.js create mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/get-pages.js create mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/index.js create mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/progressCalculator.js create mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/wait-creation.js create mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/fileCounter.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/fileCounter.js new file mode 100644 index 0000000000..c38c7aced6 --- /dev/null +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/fileCounter.js @@ -0,0 +1,13 @@ +import fs from 'fs'; + +/** + * Counts the number of files in a given directory. + * @param {string} directoryPath + * @returns {Promise} + */ +export async function countFiles(directoryPath) { + const files = await fs.promises.readdir(directoryPath); + // const fileCount = files.filter(file => fs.statSync(path.join(directoryPath, file)).isFile()).length; + const fileCount = files.length; + return fileCount; +} \ No newline at end of file diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/get-pages.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/get-pages.js new file mode 100644 index 0000000000..aec0f9ac70 --- /dev/null +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/get-pages.js @@ -0,0 +1,19 @@ +import fs from 'fs/promises'; +import path from 'path'; + +export const getPagesDir = async (/** @type {string} */ pagesDir) => { + const content = await fs.readdir(pagesDir); + console.log(path.resolve(pagesDir)) + + const markdownPages = [] + for (const fileName of content) { + const filePath = path.join(pagesDir, fileName) + const fileStat = await fs.stat(filePath) + + if (fileStat.isFile() && path.extname(filePath) === ".md") { + markdownPages.push(filePath) + } + } + + return markdownPages.length +} diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/index.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/index.js new file mode 100644 index 0000000000..a38a565709 --- /dev/null +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/index.js @@ -0,0 +1,28 @@ +import { getPagesDir } from './get-pages.js'; +import { watchDirectory } from './watcher.js'; + +/** @type {import("vite").Plugin} */ +export const verboseLogs = { + name: 'evidence:verbose-logs', + + buildStart() { + const directoryPath = '.svelte-kit/output/prerendered/pages'; + getPagesDir('../../pages') + .then((totalExpectedFiles) => { + console.log('Build started'); + watchDirectory( + directoryPath, + totalExpectedFiles, + (/** @type {string} */ progress) => { + console.log(`Build Progress: ${progress}%`); + }, + () => { + console.log('Build completed.'); + } + ); + }) + .catch((error) => { + console.error('Error:', error); + }); + } +}; diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/progressCalculator.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/progressCalculator.js new file mode 100644 index 0000000000..73098d4500 --- /dev/null +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/progressCalculator.js @@ -0,0 +1,10 @@ +/** + * Calculates the percentage of completion based on current and total values. + * @param {number} current The current progress value. + * @param {number} total The total value for completion. + * @returns {number} The percentage of completion. + */ +export function calculateProgress(current, total) { + if (total === 0) return 0; + return Math.round((current / total) * 100); +} \ No newline at end of file diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/wait-creation.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/wait-creation.js new file mode 100644 index 0000000000..e1f850a734 --- /dev/null +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/wait-creation.js @@ -0,0 +1,29 @@ +import fs from 'fs/promises' + +/** + * Waits for the specified directory to be created. + * @param {string} directoryPath + * @param {number} maxAttempts + * @param {number} intervalMs + * @returns {Promise} + */ +export async function waitForDirectoryCreation(directoryPath, maxAttempts = 30, intervalMs = 3000) { + let attempts = 0; + while (attempts < maxAttempts) { + try { + const dirExists = await (await fs.stat(directoryPath)).isDirectory() + if (dirExists) { + console.log(`Directory ${directoryPath} exists.`); + return; + } else { + console.log("dir not found") + } + } catch (error) { + console.log("waiting for the dir to be created") + } + attempts++; + await new Promise(resolve => setTimeout(resolve, intervalMs)); + } + + throw new Error(`Directory ${directoryPath} not created within the specified time.`); +} \ No newline at end of file diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js new file mode 100644 index 0000000000..f8cacd2401 --- /dev/null +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js @@ -0,0 +1,45 @@ +import chokidar from 'chokidar'; +import { countFiles } from './fileCounter.js'; +import { calculateProgress } from './progressCalculator.js'; +import { waitForDirectoryCreation } from './wait-creation.js'; + +/** + * Watches a directory and calculates the progress of files being added. + * @param {string} directoryPath + * @param {number} totalFiles + * @param {Function} onProgressUpdate + * @param {Function} onStopCallback + */ +export async function watchDirectory(directoryPath, totalFiles, onProgressUpdate, onStopCallback) { + await waitForDirectoryCreation(directoryPath); + + const currentFiles = await countFiles(directoryPath); + const initialProgress = calculateProgress(currentFiles, totalFiles); + + if (initialProgress >= 100) { + console.log('Progress already at 100% or greater. Exiting watcher.'); + if (typeof onStopCallback === 'function') { + onStopCallback(); + } + return; + } + + const watcher = chokidar.watch(directoryPath, { ignoreInitial: true }); + + async function updateProgress() { + const currentFiles = await countFiles(directoryPath); + const progress = calculateProgress(currentFiles, totalFiles); + onProgressUpdate(progress); + + if (progress >= 100) { + watcher.close(); + + if (typeof onStopCallback === 'function') { + onStopCallback(); + } + } + } + + watcher.on('add', updateProgress); + watcher.on('addDir', updateProgress); +} diff --git a/packages/lib/plugin-connector/src/build-plugins/vite.js b/packages/lib/plugin-connector/src/build-plugins/vite.js index ed2fab1ab2..47d463aa83 100644 --- a/packages/lib/plugin-connector/src/build-plugins/vite.js +++ b/packages/lib/plugin-connector/src/build-plugins/vite.js @@ -1,7 +1,8 @@ import { queryDirectoryHmr } from './query-directory-hmr.js'; import { sourceQueryHMR } from './source-query-hmr.js'; +import { verboseLogs } from './verbose-logs/index.js'; /** @type {() => import("vite").UserConfig["plugins"]} */ export const evidenceVitePlugin = () => { - return [sourceQueryHMR(), queryDirectoryHmr]; + return [sourceQueryHMR(), queryDirectoryHmr, verboseLogs]; }; From 38beb5863f81655fa6f64fb5d12b2a07d1b2640b Mon Sep 17 00:00:00 2001 From: phukon Date: Sun, 7 Apr 2024 23:13:48 +0000 Subject: [PATCH 2/3] feat: calculate template pages dir depth and track everything with one watcher --- .../build-plugins/verbose-logs/fileCounter.js | 13 -- .../verbose-logs/findTemplatePagesPaths.js | 25 +++ .../build-plugins/verbose-logs/get-pages.js | 19 --- .../src/build-plugins/verbose-logs/index.js | 27 ++-- .../verbose-logs/progressCalculator.js | 10 -- .../verbose-logs/wait-creation.js | 55 ++++--- .../src/build-plugins/verbose-logs/watcher.js | 151 +++++++++++++----- 7 files changed, 184 insertions(+), 116 deletions(-) delete mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/fileCounter.js create mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/findTemplatePagesPaths.js delete mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/get-pages.js delete mode 100644 packages/lib/plugin-connector/src/build-plugins/verbose-logs/progressCalculator.js diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/fileCounter.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/fileCounter.js deleted file mode 100644 index c38c7aced6..0000000000 --- a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/fileCounter.js +++ /dev/null @@ -1,13 +0,0 @@ -import fs from 'fs'; - -/** - * Counts the number of files in a given directory. - * @param {string} directoryPath - * @returns {Promise} - */ -export async function countFiles(directoryPath) { - const files = await fs.promises.readdir(directoryPath); - // const fileCount = files.filter(file => fs.statSync(path.join(directoryPath, file)).isFile()).length; - const fileCount = files.length; - return fileCount; -} \ No newline at end of file diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/findTemplatePagesPaths.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/findTemplatePagesPaths.js new file mode 100644 index 0000000000..377a16c160 --- /dev/null +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/findTemplatePagesPaths.js @@ -0,0 +1,25 @@ +import fs from 'fs'; +import path from 'path'; +/** + * Recursively finds template pages in a given directory and saves their paths to a set. + * @param {string} directoryPath + * @returns {Promise>} Set containing paths to template pages + */ +export const findTemplatePagesPaths = async (directoryPath) => { + const templatePagePaths = new Set(); + const files = await fs.promises.readdir(directoryPath); + + for (const file of files) { + const filePath = path.join(directoryPath, file); + const fileStat = await fs.promises.stat(filePath); + + if (fileStat.isDirectory()) { + const nestedTemplatePagePaths = await findTemplatePagesPaths(filePath); + nestedTemplatePagePaths.forEach((absolutePath) => templatePagePaths.add(absolutePath)); + } else if (file.match(/^\[(.*?)\]\.md$/)) { + templatePagePaths.add(filePath); + } + } + + return templatePagePaths; +}; diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/get-pages.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/get-pages.js deleted file mode 100644 index aec0f9ac70..0000000000 --- a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/get-pages.js +++ /dev/null @@ -1,19 +0,0 @@ -import fs from 'fs/promises'; -import path from 'path'; - -export const getPagesDir = async (/** @type {string} */ pagesDir) => { - const content = await fs.readdir(pagesDir); - console.log(path.resolve(pagesDir)) - - const markdownPages = [] - for (const fileName of content) { - const filePath = path.join(pagesDir, fileName) - const fileStat = await fs.stat(filePath) - - if (fileStat.isFile() && path.extname(filePath) === ".md") { - markdownPages.push(filePath) - } - } - - return markdownPages.length -} diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/index.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/index.js index a38a565709..ff48c5cdb3 100644 --- a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/index.js +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/index.js @@ -1,25 +1,20 @@ -import { getPagesDir } from './get-pages.js'; +import { findTemplatePagesPaths } from './findTemplatePagesPaths.js'; import { watchDirectory } from './watcher.js'; -/** @type {import("vite").Plugin} */ +/** + * Crawls the pages dir and checks for the total no. of md files. + * Then crawls the built pages in .evidence and checks for the dirs + * Compares and shows percentage complete logs. + * @type {import("vite").Plugin} + */ export const verboseLogs = { name: 'evidence:verbose-logs', - + buildStart() { const directoryPath = '.svelte-kit/output/prerendered/pages'; - getPagesDir('../../pages') - .then((totalExpectedFiles) => { - console.log('Build started'); - watchDirectory( - directoryPath, - totalExpectedFiles, - (/** @type {string} */ progress) => { - console.log(`Build Progress: ${progress}%`); - }, - () => { - console.log('Build completed.'); - } - ); + findTemplatePagesPaths('../../pages') + .then((dirs) => { + watchDirectory(directoryPath, dirs); }) .catch((error) => { console.error('Error:', error); diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/progressCalculator.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/progressCalculator.js deleted file mode 100644 index 73098d4500..0000000000 --- a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/progressCalculator.js +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Calculates the percentage of completion based on current and total values. - * @param {number} current The current progress value. - * @param {number} total The total value for completion. - * @returns {number} The percentage of completion. - */ -export function calculateProgress(current, total) { - if (total === 0) return 0; - return Math.round((current / total) * 100); -} \ No newline at end of file diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/wait-creation.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/wait-creation.js index e1f850a734..ec7e392e82 100644 --- a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/wait-creation.js +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/wait-creation.js @@ -1,4 +1,22 @@ -import fs from 'fs/promises' +import fs from 'fs/promises'; + +/** + * Checks if a directory exists. + * @param {string} directoryPath + * @returns {Promise} + */ +async function directoryExists(directoryPath) { + try { + const stats = await fs.stat(directoryPath); + return stats.isDirectory(); + } catch (error) { + // @ts-ignore + if (error.code === 'ENOENT') { + return false; + } + throw error; + } +} /** * Waits for the specified directory to be created. @@ -7,23 +25,20 @@ import fs from 'fs/promises' * @param {number} intervalMs * @returns {Promise} */ -export async function waitForDirectoryCreation(directoryPath, maxAttempts = 30, intervalMs = 3000) { - let attempts = 0; - while (attempts < maxAttempts) { - try { - const dirExists = await (await fs.stat(directoryPath)).isDirectory() - if (dirExists) { - console.log(`Directory ${directoryPath} exists.`); - return; - } else { - console.log("dir not found") - } - } catch (error) { - console.log("waiting for the dir to be created") - } - attempts++; - await new Promise(resolve => setTimeout(resolve, intervalMs)); - } +export async function waitForDirectoryCreation(directoryPath, maxAttempts = 60, intervalMs = 3000) { + let attempts = 0; + while (attempts < maxAttempts) { + const dirExists = await directoryExists(directoryPath); + if (dirExists) { + // console.log(`\nMonitoring pages build progress.`); + return; + } + // } else { + // console.log('\nDirectory not found, retrying...'); + // } + attempts++; + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } - throw new Error(`Directory ${directoryPath} not created within the specified time.`); -} \ No newline at end of file + throw new Error(`\nDirectory ${directoryPath} not created within the specified time.`); +} diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js index f8cacd2401..ff830f4deb 100644 --- a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js @@ -1,45 +1,120 @@ +import path from 'path'; import chokidar from 'chokidar'; -import { countFiles } from './fileCounter.js'; -import { calculateProgress } from './progressCalculator.js'; import { waitForDirectoryCreation } from './wait-creation.js'; +/** + * Calculates the depth of a directory path + * @param {string} dirPath - The directory path + * @returns {number} + */ +function calculateDepth(dirPath) { + const segments = dirPath.split(path.sep).filter((segment) => segment !== ''); + if (segments.length === 0) { + return 0; + } + return segments.length - 1; +} + +/** + * Calculate the max depth among an array of directory paths + * @param {string[]} dirPaths + * @returns {number} - The highest depth + */ +function getMaxDepth(dirPaths) { + let maxDepth = -1; + + for (const dirPath of dirPaths) { + const depth = calculateDepth(dirPath); + if (depth > maxDepth) { + maxDepth = depth; + } + } + + return maxDepth; +} + /** * Watches a directory and calculates the progress of files being added. - * @param {string} directoryPath - * @param {number} totalFiles - * @param {Function} onProgressUpdate - * @param {Function} onStopCallback + * @param {string} directoryPath - The path to the directory to watch. + * @param {Set} dirs - The set of directories to watch. */ -export async function watchDirectory(directoryPath, totalFiles, onProgressUpdate, onStopCallback) { - await waitForDirectoryCreation(directoryPath); - - const currentFiles = await countFiles(directoryPath); - const initialProgress = calculateProgress(currentFiles, totalFiles); - - if (initialProgress >= 100) { - console.log('Progress already at 100% or greater. Exiting watcher.'); - if (typeof onStopCallback === 'function') { - onStopCallback(); - } - return; - } - - const watcher = chokidar.watch(directoryPath, { ignoreInitial: true }); - - async function updateProgress() { - const currentFiles = await countFiles(directoryPath); - const progress = calculateProgress(currentFiles, totalFiles); - onProgressUpdate(progress); - - if (progress >= 100) { - watcher.close(); - - if (typeof onStopCallback === 'function') { - onStopCallback(); - } - } - } - - watcher.on('add', updateProgress); - watcher.on('addDir', updateProgress); +export async function watchDirectory(directoryPath, dirs) { + /** + * @type {string | number | NodeJS.Timeout | undefined} + */ + let timeout; + /** + * @type {string | number | NodeJS.Timeout | undefined} + */ + let masterTimeout; + + /** + * @type {number} + */ + let totalCount = 0; + + /** + * @type {chokidar.FSWatcher | undefined} + */ + let watcher; + + let done = false; + await waitForDirectoryCreation(directoryPath); + try { + const pathArrays = Array.from(dirs).map((p) => { + const pathArray = p.split(path.sep); + pathArray.pop(); // removing the template page name + return pathArray.slice(3); // removing target dir top paths + }); + const joinedPaths = pathArrays.map((p) => path.sep + path.join(...p)); + const depth = getMaxDepth(joinedPaths); + watcher = chokidar.watch(directoryPath, { ignoreInitial: true, depth: depth + 1 }); + /** + * Resets the timeout to 15 seconds. + * The timer will only start after the target directory has been + * created, therefore the process will not be killed prematurely. + */ + const resetTimeout = () => { + clearTimeout(timeout); + timeout = setTimeout(() => { + watcher && watcher.close(); + done = true; + }, 10000); + }; + + /** + * Resets the master timeout to ensure the plugin's timeout mechanism functions as expected. + * + * Sometimes, plugins may run again, after the pages have been built. + * If this happens, the watcher may not register any changes, and the "done" state will never be true, + * even after the pages have been built. This function is called as a redundancy measure to return control + * and not be stuck forever. + */ + const resetMasterTimeout = () => { + clearTimeout(masterTimeout); + masterTimeout = setTimeout(() => { + watcher && watcher.close(); + done = true; + }, 65000); + }; + + resetMasterTimeout(); + watcher.on('addDir', async () => { + resetTimeout(); + resetMasterTimeout(); + totalCount++; + console.clear(); // to clear the terminal and not fill the terminal with too many logs + console.log('\u001b[1m\u001b[34m Building template pages: %d+\u001b[0m', totalCount); + console.log('Please wait...'); + + if (done && watcher) { + watcher.close(); + console.log('closing watcher', done); + return; + } + }); + } catch (err) { + console.error('Error:', err); + return; + } } From d81f93fc53bcf852577c6a167a9abf9b61889150 Mon Sep 17 00:00:00 2001 From: phukon Date: Sun, 14 Apr 2024 09:42:37 +0000 Subject: [PATCH 3/3] perf: use a single timer but with longer period --- .../src/build-plugins/verbose-logs/watcher.js | 28 ++++--------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js index ff830f4deb..d36499d3d5 100644 --- a/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js +++ b/packages/lib/plugin-connector/src/build-plugins/verbose-logs/watcher.js @@ -46,7 +46,6 @@ export async function watchDirectory(directoryPath, dirs) { /** * @type {string | number | NodeJS.Timeout | undefined} */ - let masterTimeout; /** * @type {number} @@ -69,39 +68,24 @@ export async function watchDirectory(directoryPath, dirs) { const joinedPaths = pathArrays.map((p) => path.sep + path.join(...p)); const depth = getMaxDepth(joinedPaths); watcher = chokidar.watch(directoryPath, { ignoreInitial: true, depth: depth + 1 }); - /** - * Resets the timeout to 15 seconds. - * The timer will only start after the target directory has been - * created, therefore the process will not be killed prematurely. - */ + + // To trigger set the done state after 60 seconds of inactivity. const resetTimeout = () => { clearTimeout(timeout); timeout = setTimeout(() => { watcher && watcher.close(); done = true; - }, 10000); + }, 60000); }; /** - * Resets the master timeout to ensure the plugin's timeout mechanism functions as expected. - * - * Sometimes, plugins may run again, after the pages have been built. + * Sometimes, plugins runs again after the pages have been built. * If this happens, the watcher may not register any changes, and the "done" state will never be true, - * even after the pages have been built. This function is called as a redundancy measure to return control - * and not be stuck forever. + * even after the pages have been built. Hence setting up the timer before the watcher as well. */ - const resetMasterTimeout = () => { - clearTimeout(masterTimeout); - masterTimeout = setTimeout(() => { - watcher && watcher.close(); - done = true; - }, 65000); - }; - - resetMasterTimeout(); + resetTimeout(); watcher.on('addDir', async () => { resetTimeout(); - resetMasterTimeout(); totalCount++; console.clear(); // to clear the terminal and not fill the terminal with too many logs console.log('\u001b[1m\u001b[34m Building template pages: %d+\u001b[0m', totalCount);