feat: jsx migration script (#869)

* feat: jsx-migration script (wip)

* fix: deps

* feat: logging

* fix: handle extensionless and duplicate imports

* fix: avoid IDing types as JSX, handle .component files better, flag for extensionless imports

* refactor: use babel for AST parsing

* fix: better logging

* fix: improve d2 config handling

* feat: handle export statements

* feat: custom glob pattern

* chore: clean-up

* docs: jsx-migration

* refactor: add script to 'migrate' namespace

* fix: reporter text styles

* fix: update caniuse-lite to remove warning

* docs: update for new namespace

cli/package.json

@@ -25,6 +25,7 @@
         "@babel/plugin-proposal-class-properties": "^7.8.3",
         "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3",
         "@babel/plugin-proposal-optional-chaining": "^7.8.3",
+        "@babel/plugin-syntax-flow": "^7.24.7",
         "@babel/preset-env": "^7.14.7",
         "@babel/preset-react": "^7.0.0",
         "@babel/preset-typescript": "^7.6.0",
@@ -45,6 +46,7 @@
         "detect-port": "^1.3.0",
         "dotenv": "^8.1.0",
         "dotenv-expand": "^5.1.0",
+        "fast-glob": "^3.3.2",
         "file-loader": "^6.2.0",
         "form-data": "^3.0.0",
         "fs-extra": "^8.1.0",

cli/src/commands/migrate.js

@@ -0,0 +1,6 @@
+const { namespace } = require('@dhis2/cli-helpers-engine')
+
+module.exports = namespace('migrate', {
+    desc: 'Scripts to make changes to DHIS2 apps',
+    builder: (yargs) => yargs.commandDir('migrate'),
+})

cli/src/commands/migrate/js-to-jsx.js

@@ -0,0 +1,266 @@
+const fs = require('fs/promises')
+const path = require('path')
+const babel = require('@babel/core')
+const { reporter /* chalk */ } = require('@dhis2/cli-helpers-engine')
+const fg = require('fast-glob')
+
+// These are the plugins needed to parse JS with various syntaxes --
+// typescript shouldn't be needed
+const babelParseOptions = {
+    // could just use jsx syntax parser, but this is already a dep of CLI
+    presets: ['@babel/preset-react'],
+    // just need syntax parser here
+    plugins: ['@babel/plugin-syntax-flow'],
+}
+
+const isJsxInFile = async (filepath) => {
+    const code = await fs.readFile(filepath, { encoding: 'utf8' })
+    try {
+        const ast = await babel.parseAsync(code, babelParseOptions)
+
+        let isJsx = false
+        babel.traverse(ast, {
+            // Triggers for any JSX-type node (JSXElement, JSXAttribute, etc)
+            JSX: (path) => {
+                isJsx = true
+                path.stop() // done here; stop traversing
+            },
+        })
+
+        return isJsx
+    } catch (err) {
+        console.log(err)
+        return false
+    }
+}
+
+const renameFile = async (filepath) => {
+    const newPath = filepath.concat('x') // Add 'x' to the end to make it 'jsx'
+    reporter.debug(`Renaming ${filepath} to ${newPath}`)
+    await fs.rename(filepath, newPath)
+}
+
+/**
+ * For JS imports, this will handle imports either with or without a .js
+ * extension, such that the result ends with .jsx if the target file has been
+ * renamed
+ * Files without extension are updated by default since some linting rules give
+ * `import/no-unresolved` errors after switching to JSX if imports don't use
+ * an extension
+ * If `skipUpdatingImportsWithoutExtension` is set, imports without an extension
+ * will be left as-is
+ */
+const resolveImportSource = ({
+    filepath,
+    importSource,
+    renamedFiles,
+    skipUpdatingImportsWithoutExtension,
+}) => {
+    // This doesn't handle files with an extension other than .js,
+    // since they won't need updating
+    const importSourceHasExtension = importSource.endsWith('.js')
+    if (skipUpdatingImportsWithoutExtension && !importSourceHasExtension) {
+        return importSource
+    }
+
+    // We'll need an extension to match with the renamed files Set
+    const importSourceWithExtension = importSourceHasExtension
+        ? importSource
+        : importSource + '.js'
+    // get the full path of the imported file from the cwd
+    const importPathFromCwd = path.join(
+        filepath,
+        '..',
+        importSourceWithExtension
+    )
+
+    const isRenamed = renamedFiles.has(importPathFromCwd)
+    return isRenamed ? importSourceWithExtension + 'x' : importSource
+}
+
+const updateImports = async ({
+    filepath,
+    renamedFiles,
+    skipUpdatingImportsWithoutExtension,
+}) => {
+    const code = await fs.readFile(filepath, { encoding: 'utf8' })
+    reporter.debug(`Parsing ${filepath}`)
+
+    try {
+        const ast = await babel.parseAsync(code, babelParseOptions)
+
+        let newCode = code
+        let contentUpdated = false
+        babel.traverse(ast, {
+            // Triggers on imports and exports, the latter for cases like
+            // `export * from './file.js'`
+            'ImportDeclaration|ExportDeclaration': (astPath) => {
+                if (!astPath.node.source) {
+                    return // for exports from this file itself
+                }
+
+                const importSource = astPath.node.source.value
+                if (!importSource.startsWith('.')) {
+                    return // not a relative import
+                }
+
+                const newImportSource = resolveImportSource({
+                    filepath,
+                    importSource,
+                    renamedFiles,
+                    skipUpdatingImportsWithoutExtension,
+                })
+
+                // Since generating code from babel doesn't respect formatting,
+                // update imports with just string replacement
+                if (newImportSource !== importSource) {
+                    // updating & replacing the raw value, which includes quotes,
+                    // ends up being more precise and avoids side effects
+                    const rawImportSource = astPath.node.source.extra.raw
+                    const newRawImportSource = rawImportSource.replace(
+                        importSource,
+                        newImportSource
+                    )
+                    reporter.debug(
+                        `    Replacing ${importSource} => ${newImportSource}`
+                    )
+                    newCode = newCode.replace(
+                        rawImportSource,
+                        newRawImportSource
+                    )
+                    contentUpdated = true
+                }
+            },
+        })
+
+        if (contentUpdated) {
+            await fs.writeFile(filepath, newCode)
+        }
+        return contentUpdated
+    } catch (err) {
+        console.log(err)
+        return false
+    }
+}
+
+const validateGlobString = (glob) => {
+    if (!glob.endsWith('.js')) {
+        throw new Error('Glob string must end with .js')
+    }
+}
+const defaultGlobString = 'src/**/*.js'
+// in case a custom glob string includes node_modules somewhere:
+const globOptions = { ignore: ['**/node_modules/**'] }
+
+const handler = async ({
+    globString = defaultGlobString,
+    skipUpdatingImportsWithoutExtension,
+}) => {
+    validateGlobString(globString)
+
+    // 1. Search each JS file for JSX syntax
+    // If found, 2) Rename (add 'x' to the end) and 2) add path to a Set
+    reporter.info(`Using glob ${globString}`)
+    const globMatches = await fg.glob(globString, globOptions)
+    reporter.info(`Searching for JSX in ${globMatches.length} files...`)
+    const renamedFiles = new Set()
+    await Promise.all(
+        globMatches.map(async (matchPath) => {
+            const jsxIsInFile = await isJsxInFile(matchPath)
+            if (jsxIsInFile) {
+                await renameFile(matchPath, renamedFiles)
+                renamedFiles.add(matchPath)
+            }
+        })
+    )
+    reporter.print(`Renamed ${renamedFiles.size} file(s)`)
+
+    // 2. Go through each file again for imports
+    // (Run glob again and include .jsx because some files have been renamed)
+    // If there's a local file import, check to see if it matches
+    // a renamed item in the set. If so, rewrite the new extension
+    const globMatches2 = await fg.glob(globString + '(|x)', globOptions)
+    reporter.info(`Scanning ${globMatches2.length} files to update imports...`)
+    let fileUpdatedCount = 0
+    await Promise.all(
+        globMatches2.map(async (matchPath) => {
+            const importsAreUpdated = await updateImports({
+                filepath: matchPath,
+                renamedFiles,
+                skipUpdatingImportsWithoutExtension,
+            })
+            if (importsAreUpdated) {
+                fileUpdatedCount++
+            }
+        })
+    )
+    reporter.print(`Updated imports in ${fileUpdatedCount} file(s)`)
+
+    // 3. Update d2.config.js
+    const d2ConfigPath = path.join(process.cwd(), 'd2.config.js')
+    reporter.info('Checking d2.config.js for entry points to update...')
+    reporter.debug(`d2 config path: ${d2ConfigPath}`)
+
+    // Read d2 config as JS for easy access to entryPoint strings
+    let entryPoints
+    try {
+        const d2Config = require(d2ConfigPath)
+        entryPoints = d2Config.entryPoints
+    } catch (err) {
+        reporter.warn(`Did not find d2.config.js at ${d2ConfigPath}; finishing`)
+        return
+    }
+
+    const d2ConfigContents = await fs.readFile(d2ConfigPath, {
+        encoding: 'utf8',
+    })
+    let newD2ConfigContents = d2ConfigContents
+    let configContentUpdated = false
+    Object.values(entryPoints).forEach((entryPoint) => {
+        const newEntryPointSource = resolveImportSource({
+            filepath: 'd2.config.js',
+            importSource: entryPoint,
+            renamedFiles,
+            skipUpdatingImportsWithoutExtension,
+        })
+        if (newEntryPointSource !== entryPoint) {
+            newD2ConfigContents = newD2ConfigContents.replace(
+                entryPoint,
+                newEntryPointSource
+            )
+            configContentUpdated = true
+            reporter.debug(
+                `Updating entry point ${entryPoint} => ${newEntryPointSource}`
+            )
+        }
+    })
+
+    if (configContentUpdated) {
+        await fs.writeFile(d2ConfigPath, newD2ConfigContents)
+        reporter.print('Updated d2.config.js entry points')
+    } else {
+        reporter.print('No entry points updated')
+    }
+}
+
+const command = {
+    command: 'js-to-jsx',
+    desc: 'Renames .js files that include JSX to .jsx. Also handles file imports and d2.config.js',
+    builder: {
+        skipUpdatingImportsWithoutExtension: {
+            description:
+                "Normally, this script will update `import './App'` to `import './App.jsx'`. Use this flag to skip adding the extension in this case. Imports that already end with .js will still be updated to .jsx",
+            type: 'boolean',
+            default: false,
+        },
+        globString: {
+            description:
+                'Glob string to use for finding files to parse, rename, and update imports. It will be manipulated by the script, so it must end with .js, and make sure to use quotes around this argument to keep it a string',
+            type: 'string',
+            default: defaultGlobString,
+        },
+    },
+    handler,
+}
+
+module.exports = command

docs/_sidebar.md

@@ -9,6 +9,8 @@
     -   [`d2-app-scripts pack`](scripts/pack.md)
     -   [`d2-app-scripts deploy`](scripts/deploy.md)
     -   [`d2-app-scripts publish`](scripts/publish.md)
+    -   [**Migrate**](scripts/migrate)
+        -   [`d2-app-scripts migrate js-to-jsx`](scripts/migrate/js-to-jsx.md)
 -   [**Configuration**](config.md)
     -   [Types - `app`, `lib`](config/types)
     -   [`d2.config.js` Reference](config/d2-config-js-reference.md)

docs/scripts/migrate/js-to-jsx.md

@@ -0,0 +1,75 @@
+# d2-app-scripts migrate js-to-jsx
+
+Converts files with `.js` extensions to `.jsx` if the file contains JSX syntax. This is intended as a helper for moving `@dhis2/cli-app-scripts` to Vite, which prefers files to be named as such to avoid unnecessarily parsing vanilla JS files for JSX syntax.
+
+## Example
+
+This should usually be run from the root directory of a project.
+
+```sh
+yarn d2-app-scripts migrate js-to-jsx
+```
+
+By default, this will crawl through each `.js` file in the `src` directory (using the glob `src/**/*.js`), look for JSX syntax in the file, then rename the file to use a `.jsx` extension if appropriate.
+
+Then, it will crawl through all `.js` _and_ `.jsx` file in `src` and update file imports to match the newly renamed files. **By default, this will update imports without a file extension**, e.g. `import Component from './Component'` => `import Component from './Component.jsx'`. This is because, in testing, updating files to `.jsx` extensions without updating the imports ends up causing linting errors. Functionally, the app will still work without extensions on imports though; Vite handles it. If you don't want to update imports without extensions, you can use the `--skipUpdatingImportsWithoutExtension` flag when running this script. Imports that use a `.js` extension will be updated to `.jsx` either way.
+
+Lastly, the script will check `d2.config.js` in the CWD for entry points to update if the respective files have been renamed.
+
+## Tips
+
+This may update a _lot_ of files; be prepared with your source control to undo changes if needed. In VSCode, for example, there is an feature in the Source Control UI to "Discard All Changes" from unstaged files. Before running the script, stage the files you want to keep around, then run the script. If the outcome isn't what you want, you can use the "Discard All Changes" option to undo them easily.
+
+Note that renamed files are only kept track of during script execution. If, for example, you run the script, then you want to redo it with the `--skipUpdatingImportsWithoutExtension` flag, it's best to undo all the renamed files before running the script again.
+
+### `--globString`
+
+The script will crawl through files using the `src/**/*.js` glob by default. If you want to crawl different directories, for example to migrate smaller pieces of a project at a time, you can specify a custom glob when running the script.
+
+Since imports will only be updated within the scope of that glob, a directory that exports its contents through an `index.js` file is an ideal choice.
+
+Example:
+
+```sh
+yarn d2-app-scripts migrate js-to-jsx --globString "src/components/**/*.js"
+```
+
+Since the glob string will be reused and manipulated by the script, make sure to use quotes around the argument so that the shell doesn't handle it as a normal glob.
+
+Contents of `node_modules` directories will always be ignored. `d2.config.js` will still be sought out in the CWD, but won't cause an error if one is not found.
+
+## Usage
+
+```sh
+> d2-app-scripts migrate js-to-jsx --help
+d2-app-scripts migrate js-to-jsx
+
+Renames .js files that include JSX to .jsx. Also handles file imports and
+d2.config.js
+
+Global Options:
+  -h, --help     Show help                                             [boolean]
+  -v, --version  Show version number                                   [boolean]
+  --verbose      Enable verbose messages                               [boolean]
+  --debug        Enable debug messages                                 [boolean]
+  --quiet        Enable quiet mode                                     [boolean]
+  --config       Path to JSON config file
+
+Options:
+  --cwd                                  working directory to use (defaults to
+                                         cwd)
+  --skipUpdatingImportsWithoutExtension  Normally, this script will update
+                                         `import './App'` to `import
+                                         './App.jsx'`. Use this flag to skip
+                                         adding the extension in this case.
+                                         Imports that already end with .js will
+                                         still be updated to .jsx
+                                                      [boolean] [default: false]
+  --globString                           Glob string to use for finding files to
+                                         parse, rename, and update imports. It
+                                         will be manipulated by the script, so
+                                         it must end with .js, and make sure to
+                                         use quotes around this argument to keep
+                                         it a string
+                                               [string] [default: "src/**/*.js"]
+```