diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 2017735dc758..325e3c2608f5 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -16,6 +16,79 @@ to all development within the OpenSearch Dashboards project. Please make sure to - [Accessibility developer guide (EUI Docs)](https://elastic.github.io/eui/#/guidelines/accessibility) - [SASS developer guide (EUI Docs)](https://elastic.github.io/eui/#/guidelines/sass) +## Getting Started + +If you would like to install and run this project, please see the [Downloads Page](https://opensearch.org/downloads.html). + +#### Prerequisites + +You need to have an OpenSearch server up and running to be able to run OpenSearch +Dashboards. The easiest way to do it is [using Docker](https://opensearch.org/docs/opensearch/install/docker). + +We recommend using [Node Version Manager](https://github.com/nvm-sh/nvm) to install +the node version we need. + +### Bootstrap OpenSearch Dashboards + +While OpenSearch is starting, you can already bootstrap OpenSearch Dashboards: +```bash +$ git clone https://github.com/opensearch-project/OpenSearch-Dashboards.git +$ cd OpenSearch-Dashboards +$ nvm use +$ npm i -g yarn +$ yarn osd bootstrap # This command will also install npm dependencies +``` + +### Configure OpenSearch Dashboards + +*This step is only mandatory if you have https/authentication enabled, or if you use the OpenSearch Docker image in its default configuration.* + +Once the bootstrap of OpenSearch Dashboards is finished, you need to apply some +changes to `opensearch_dashboards.yml` in order to connect to OpenSearch. + +```yml +opensearch.hosts: ["https://localhost:9200"] +opensearch.username: "admin" # Default username on the docker image +opensearch.password: "admin" # Default password on the docker image +opensearch.ssl.verificationMode: none +``` + +### Run OpenSearch Dashboards + +After you've installed OpenSearch Dashboards and configured it, you can start +the development server: +```bash +$ yarn start +``` + +When the server is up and ready, click on the link displayed in your terminal to +access it. + +### Building the artifacts + +To build the archives for each platform, run the following: + +``` +yarn build --skip-os-packages +``` + +If you want to build a specific platform, pass the platform flag after `yarn build-platform`. For example, to build darwin x64, run the following: + +``` +yarn build-platform --darwin +``` + +You could pass one or multiple flags. If you don't pass any flag, `yarn build-platform` will use your local environment. Currenly we only support `darwin` (darwin x64), `linux` (linux x64) and `linux-arm` (linux arm64). + +### Building the Docker Image + +To build the Docker image, run the following: + +``` +yarn osd bootstrap +yarn build --docker +``` + ## General ### Filenames diff --git a/package.json b/package.json index f182087df833..16a08ff62096 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "test:ftr:runner": "node scripts/functional_test_runner", "test:coverage": "grunt test:coverage", "checkLicenses": "node scripts/check_licenses --dev", + "build-platform": "node scripts/build", "build": "node scripts/build --all-platforms", "start": "node scripts/opensearch_dashboards --dev", "debug": "node --nolazy --inspect scripts/opensearch_dashboards --dev", diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts index da9f05bc7deb..c32a12341eb5 100644 --- a/src/dev/build/args.test.ts +++ b/src/dev/build/args.test.ts @@ -53,6 +53,89 @@ it('build dist for current platform, without packages, by default', () => { "downloadFreshNode": true, "isRelease": false, "targetAllPlatforms": false, + "targetPlatforms": Object { + "darwin": false, + "linux": false, + "linuxArm": false, + }, + "versionQualifier": "", + }, + "log": , + "showHelp": false, + "unknownFlags": Array [], + } + `); +}); + +it('build dist for linux x64 platform, without packages, if --linux-x64 is passed', () => { + expect(readCliArgs(['node', 'scripts/build-platform'])).toMatchInlineSnapshot(` + Object { + "buildOptions": Object { + "createArchives": true, + "createDebPackage": false, + "createDockerPackage": false, + "createDockerUbiPackage": false, + "createRpmPackage": false, + "downloadFreshNode": true, + "isRelease": false, + "targetAllPlatforms": false, + "targetPlatforms": Object { + "darwin": false, + "linux": false, + "linuxArm": false, + }, + "versionQualifier": "", + }, + "log": , + "showHelp": false, + "unknownFlags": Array [], + } + `); +}); + +it('build dist for linux x64 platform, without packages, if --linux-arm64 is passed', () => { + expect(readCliArgs(['node', 'scripts/build-platform'])).toMatchInlineSnapshot(` + Object { + "buildOptions": Object { + "createArchives": true, + "createDebPackage": false, + "createDockerPackage": false, + "createDockerUbiPackage": false, + "createRpmPackage": false, + "downloadFreshNode": true, + "isRelease": false, + "targetAllPlatforms": false, + "targetPlatforms": Object { + "darwin": false, + "linux": false, + "linuxArm": false, + }, + "versionQualifier": "", + }, + "log": , + "showHelp": false, + "unknownFlags": Array [], + } + `); +}); + +it('build dist for linux x64 platform, without packages, if --darwin-x64 is passed', () => { + expect(readCliArgs(['node', 'scripts/build-platform'])).toMatchInlineSnapshot(` + Object { + "buildOptions": Object { + "createArchives": true, + "createDebPackage": false, + "createDockerPackage": false, + "createDockerUbiPackage": false, + "createRpmPackage": false, + "downloadFreshNode": true, + "isRelease": false, + "targetAllPlatforms": false, + "targetPlatforms": Object { + "darwin": false, + "linux": false, + "linuxArm": false, + }, "versionQualifier": "", }, "log": , @@ -74,6 +157,11 @@ it('builds packages if --all-platforms is passed', () => { "downloadFreshNode": true, "isRelease": false, "targetAllPlatforms": true, + "targetPlatforms": Object { + "darwin": false, + "linux": false, + "linuxArm": false, + }, "versionQualifier": "", }, "log": , @@ -95,6 +183,11 @@ it('limits packages if --rpm passed with --all-platforms', () => { "downloadFreshNode": true, "isRelease": false, "targetAllPlatforms": true, + "targetPlatforms": Object { + "darwin": false, + "linux": false, + "linuxArm": false, + }, "versionQualifier": "", }, "log": , @@ -116,6 +209,11 @@ it('limits packages if --deb passed with --all-platforms', () => { "downloadFreshNode": true, "isRelease": false, "targetAllPlatforms": true, + "targetPlatforms": Object { + "darwin": false, + "linux": false, + "linuxArm": false, + }, "versionQualifier": "", }, "log": , @@ -138,6 +236,11 @@ it('limits packages if --docker passed with --all-platforms', () => { "downloadFreshNode": true, "isRelease": false, "targetAllPlatforms": true, + "targetPlatforms": Object { + "darwin": false, + "linux": false, + "linuxArm": false, + }, "versionQualifier": "", }, "log": , @@ -160,6 +263,11 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform "downloadFreshNode": true, "isRelease": false, "targetAllPlatforms": true, + "targetPlatforms": Object { + "darwin": false, + "linux": false, + "linuxArm": false, + }, "versionQualifier": "", }, "log": , diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts index 8d1c92269d4f..df7de0477c40 100644 --- a/src/dev/build/args.ts +++ b/src/dev/build/args.ts @@ -45,6 +45,9 @@ export function readCliArgs(argv: string[]) { 'verbose', 'debug', 'all-platforms', + 'darwin', + 'linux', + 'linux-arm', 'verbose', 'quiet', 'silent', @@ -111,6 +114,11 @@ export function readCliArgs(argv: string[]) { createDebPackage: isOsPackageDesired('deb'), createDockerPackage: isOsPackageDesired('docker'), createDockerUbiPackage: isOsPackageDesired('docker') && !Boolean(flags['skip-docker-ubi']), + targetPlatforms: { + darwin: Boolean(flags.darwin), + linux: Boolean(flags.linux), + linuxArm: Boolean(flags['linux-arm']), + }, targetAllPlatforms: Boolean(flags['all-platforms']), }; diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index a13ee871306d..39983aec1ce3 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -27,7 +27,7 @@ import { ToolingLog } from '@osd/dev-utils'; -import { Config, createRunner } from './lib'; +import { Config, createRunner, TargetPlatforms } from './lib'; import * as Tasks from './tasks'; export interface BuildOptions { @@ -40,6 +40,7 @@ export interface BuildOptions { createDockerUbiPackage: boolean; versionQualifier: string | undefined; targetAllPlatforms: boolean; + targetPlatforms: TargetPlatforms; } export async function buildDistributables(log: ToolingLog, options: BuildOptions) { diff --git a/src/dev/build/cli.ts b/src/dev/build/cli.ts index ff87b2530f93..a83ed1fabf91 100644 --- a/src/dev/build/cli.ts +++ b/src/dev/build/cli.ts @@ -55,6 +55,9 @@ if (showHelp) { --skip-archives {dim Don't produce tar/zip archives} --skip-os-packages {dim Don't produce rpm/deb/docker packages} --all-platforms {dim Produce archives for all platforms, not just this one} + --linux-x64 {dim Produce archives for only linux x64 platform} + --linux-arm64 {dim Produce archives for only linux arm64 platform} + --darwin-x64 {dim Produce archives for only darwin x64 platform} --rpm {dim Only build the rpm package} --deb {dim Only build the deb package} --docker {dim Only build the docker image} diff --git a/src/dev/build/lib/build.test.ts b/src/dev/build/lib/build.test.ts index d2ce1f3ab165..bbadd2cc0cff 100644 --- a/src/dev/build/lib/build.test.ts +++ b/src/dev/build/lib/build.test.ts @@ -36,7 +36,12 @@ expect.addSnapshotSerializer(createAbsolutePathSerializer()); const config = new Config( true, { - version: '8.0.0', + darwin: false, + linux: false, + linuxArm: false, + }, + { + version: '1.0.0', engines: { node: '*', }, @@ -49,7 +54,7 @@ const config = new Config( { buildNumber: 1234, buildSha: 'abcd1234', - buildVersion: '8.0.0', + buildVersion: '1.0.0', }, true ); @@ -93,7 +98,7 @@ describe('#resolvePath()', () => { describe('#resolvePathForPlatform()', () => { it('uses config.resolveFromRepo(), config.getBuildVersion(), and platform.getBuildName() to create path', () => { expect(build.resolvePathForPlatform(linuxPlatform, 'foo', 'bar')).toMatchInlineSnapshot( - `/build/opensearch-dashboards-8.0.0-linux-x64/foo/bar` + `/build/opensearch-dashboards-1.0.0-linux-x64/foo/bar` ); }); }); @@ -101,13 +106,13 @@ describe('#resolvePathForPlatform()', () => { describe('#getPlatformArchivePath()', () => { it('creates correct path for different platforms', () => { expect(build.getPlatformArchivePath(linuxPlatform)).toMatchInlineSnapshot( - `/target/opensearch-dashboards-8.0.0-linux-x64.tar.gz` + `/target/opensearch-dashboards-1.0.0-linux-x64.tar.gz` ); expect(build.getPlatformArchivePath(linuxArmPlatform)).toMatchInlineSnapshot( - `/target/opensearch-dashboards-8.0.0-linux-arm64.tar.gz` + `/target/opensearch-dashboards-1.0.0-linux-arm64.tar.gz` ); expect(build.getPlatformArchivePath(windowsPlatform)).toMatchInlineSnapshot( - `/target/opensearch-dashboards-8.0.0-windows-x64.zip` + `/target/opensearch-dashboards-1.0.0-windows-x64.zip` ); }); }); diff --git a/src/dev/build/lib/config.test.ts b/src/dev/build/lib/config.test.ts index 4aeb97249562..077076b58c01 100644 --- a/src/dev/build/lib/config.test.ts +++ b/src/dev/build/lib/config.test.ts @@ -45,10 +45,25 @@ const versionInfo = jest.requireMock('./version_info').getVersionInfo(); expect.addSnapshotSerializer(createAbsolutePathSerializer()); -const setup = async ({ targetAllPlatforms = true }: { targetAllPlatforms?: boolean } = {}) => { +const setup = async ({ + targetAllPlatforms = true, + targetPlatforms = { + darwin: false, + linux: false, + linuxArm: false, + }, +}: { + targetAllPlatforms?: boolean; + targetPlatforms?: { + darwin: boolean; + linux: boolean; + linuxArm: boolean; + }; +} = {}) => { return await Config.create({ isRelease: true, targetAllPlatforms, + targetPlatforms, }); }; @@ -89,6 +104,51 @@ describe('#resolveFromRepo()', () => { }); }); +describe('#hasSpecifiedPlatform', () => { + it('return true if darwin is specified', async () => { + const config = await setup({ + targetAllPlatforms: false, + targetPlatforms: { + darwin: true, + linux: false, + linuxArm: false, + }, + }); + expect(config.hasSpecifiedPlatform() === true); + }); + + it('return true if linux arm64 is specified', async () => { + const config = await setup({ + targetAllPlatforms: false, + targetPlatforms: { + darwin: false, + linux: false, + linuxArm: true, + }, + }); + expect(config.hasSpecifiedPlatform() === true); + }); + + it('return true if linux is specified', async () => { + const config = await setup({ + targetAllPlatforms: false, + targetPlatforms: { + darwin: false, + linux: true, + linuxArm: false, + }, + }); + expect(config.hasSpecifiedPlatform() === true); + }); + + it('return false if no platform is specified', async () => { + const config = await setup({ + targetAllPlatforms: true, + }); + expect(config.hasSpecifiedPlatform() === false); + }); +}); + describe('#getPlatform()', () => { it('throws error when platform does not exist', async () => { const config = await setup(); @@ -118,7 +178,7 @@ describe('#getPlatform()', () => { }); describe('#getTargetPlatforms()', () => { - it('returns an array of all platform objects', async () => { + it('returns an array of all platform objects if config.targetAllPlatforms is true', async () => { const config = await setup(); expect( config @@ -135,6 +195,95 @@ describe('#getTargetPlatforms()', () => { `); }); + it('returns just darwin x64 platform when darwin = true', async () => { + const config = await setup({ + targetAllPlatforms: false, + targetPlatforms: { + darwin: true, + linux: false, + linuxArm: false, + }, + }); + + expect( + config + .getTargetPlatforms() + .map((p) => p.getNodeArch()) + .sort() + ).toMatchInlineSnapshot(` + Array [ + "darwin-x64", + ] + `); + }); + + it('returns just linux x64 platform when linux = true', async () => { + const config = await setup({ + targetAllPlatforms: false, + targetPlatforms: { + darwin: false, + linux: true, + linuxArm: false, + }, + }); + + expect( + config + .getTargetPlatforms() + .map((p) => p.getNodeArch()) + .sort() + ).toMatchInlineSnapshot(` + Array [ + "linux-x64", + ] + `); + }); + + it('returns just linux arm64 platform when linuxArm64 = true', async () => { + const config = await setup({ + targetAllPlatforms: false, + targetPlatforms: { + darwin: false, + linux: false, + linuxArm: true, + }, + }); + + expect( + config + .getTargetPlatforms() + .map((p) => p.getNodeArch()) + .sort() + ).toMatchInlineSnapshot(` + Array [ + "linux-arm64", + ] + `); + }); + + it('returns both linux arm64 and darwin x64 platforms when linuxArm64 and darwin equal to true', async () => { + const config = await setup({ + targetAllPlatforms: false, + targetPlatforms: { + darwin: true, + linux: false, + linuxArm: true, + }, + }); + + expect( + config + .getTargetPlatforms() + .map((p) => p.getNodeArch()) + .sort() + ).toMatchInlineSnapshot(` + Array [ + "darwin-x64", + "linux-arm64", + ] + `); + }); + it('returns just this platform when targetAllPlatforms = false', async () => { const config = await setup({ targetAllPlatforms: false, diff --git a/src/dev/build/lib/config.ts b/src/dev/build/lib/config.ts index 5389c8f3465d..1abaad734140 100644 --- a/src/dev/build/lib/config.ts +++ b/src/dev/build/lib/config.ts @@ -30,11 +30,18 @@ import os from 'os'; import loadJsonFile from 'load-json-file'; import { getVersionInfo, VersionInfo } from './version_info'; -import { PlatformName, PlatformArchitecture, ALL_PLATFORMS } from './platform'; +import { + PlatformName, + PlatformArchitecture, + ALL_PLATFORMS, + Platform, + TargetPlatforms, +} from './platform'; interface Options { isRelease: boolean; targetAllPlatforms: boolean; + targetPlatforms: TargetPlatforms; versionQualifier?: string; } @@ -48,12 +55,18 @@ interface Package { } export class Config { - static async create({ isRelease, targetAllPlatforms, versionQualifier }: Options) { + static async create({ + isRelease, + targetAllPlatforms, + targetPlatforms, + versionQualifier, + }: Options) { const pkgPath = resolve(__dirname, '../../../../package.json'); const pkg: Package = loadJsonFile.sync(pkgPath); return new Config( targetAllPlatforms, + targetPlatforms, pkg, pkg.engines.node, dirname(pkgPath), @@ -68,6 +81,7 @@ export class Config { constructor( private readonly targetAllPlatforms: boolean, + private readonly targetPlatforms: TargetPlatforms, private readonly pkg: Package, private readonly nodeVersion: string, private readonly repoRoot: string, @@ -104,14 +118,32 @@ export class Config { } /** - * Return the list of Platforms we are targeting, if --this-platform flag is - * specified only the platform for this OS will be returned + * Return true if a supported Platform is specified + */ + hasSpecifiedPlatform() { + for (const platform in this.targetPlatforms) { + if (platform) return true; + } + return false; + } + + /** + * Return the list of supported Platforms, if --all-platform flag is specified. + * Return the targeted Platforms, if --target-platform flags are specified. + * Return the platform of local OS. If it is not a supported Platform, raise an error. */ getTargetPlatforms() { if (this.targetAllPlatforms) { return ALL_PLATFORMS; } + const platforms: Platform[] = []; + if (this.targetPlatforms.darwin) platforms.push(this.getPlatform('darwin', 'x64')); + if (this.targetPlatforms.linux) platforms.push(this.getPlatform('linux', 'x64')); + if (this.targetPlatforms.linuxArm) platforms.push(this.getPlatform('linux', 'arm64')); + + if (platforms.length > 0) return platforms; + return [this.getPlatformForThisOs()]; } diff --git a/src/dev/build/lib/platform.ts b/src/dev/build/lib/platform.ts index 9c37c5749776..349cb094fdba 100644 --- a/src/dev/build/lib/platform.ts +++ b/src/dev/build/lib/platform.ts @@ -28,6 +28,12 @@ export type PlatformName = 'win32' | 'darwin' | 'linux'; export type PlatformArchitecture = 'x64' | 'arm64'; +export interface TargetPlatforms { + darwin: boolean; + linuxArm: boolean; + linux: boolean; +} + export class Platform { constructor( private name: PlatformName, diff --git a/src/dev/build/tasks/clean_tasks.ts b/src/dev/build/tasks/clean_tasks.ts index 7ce662061859..47a848446aff 100644 --- a/src/dev/build/tasks/clean_tasks.ts +++ b/src/dev/build/tasks/clean_tasks.ts @@ -183,7 +183,7 @@ export const CleanExtraBinScripts: Task = { description: 'Cleaning extra bin/* scripts from platform-specific builds', async run(config, log, build) { - for (const platform of config.getNodePlatforms()) { + for (const platform of config.getTargetPlatforms()) { if (platform.isWindows()) { await deleteAll( [ diff --git a/src/dev/build/tasks/create_archives_task.ts b/src/dev/build/tasks/create_archives_task.ts index f64510e09510..1bf38936bb2b 100644 --- a/src/dev/build/tasks/create_archives_task.ts +++ b/src/dev/build/tasks/create_archives_task.ts @@ -40,7 +40,6 @@ export const CreateArchives: Task = { async run(config, log, build) { const archives = []; - // archive one at a time, parallel causes OOM sometimes for (const platform of config.getTargetPlatforms()) { const source = build.resolvePathForPlatform(platform, '.'); diff --git a/src/dev/build/tasks/nodejs/clean_node_builds_task.ts b/src/dev/build/tasks/nodejs/clean_node_builds_task.ts index d42f9d73e214..871f61d51402 100644 --- a/src/dev/build/tasks/nodejs/clean_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/clean_node_builds_task.ts @@ -31,7 +31,7 @@ export const CleanNodeBuilds: Task = { description: 'Cleaning npm from node', async run(config, log, build) { - for (const platform of config.getNodePlatforms()) { + for (const platform of config.getTargetPlatforms()) { await deleteAll( [ build.resolvePathForPlatform(platform, 'node/lib/node_modules'), diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.ts index a2d47f1d71fd..08e83e0dea3e 100644 --- a/src/dev/build/tasks/nodejs/download_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/download_node_builds_task.ts @@ -35,7 +35,7 @@ export const DownloadNodeBuilds: GlobalTask = { async run(config, log) { const shasums = await getNodeShasums(log, config.getNodeVersion()); await Promise.all( - config.getNodePlatforms().map(async (platform) => { + config.getTargetPlatforms().map(async (platform) => { const { url, downloadPath, downloadName } = getNodeDownloadInfo(config, platform); await download({ log, diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.ts index 2e0b9521240c..6bb5fef4ec0e 100644 --- a/src/dev/build/tasks/nodejs/extract_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/extract_node_builds_task.ts @@ -35,7 +35,7 @@ export const ExtractNodeBuilds: GlobalTask = { description: 'Extracting node.js builds for all platforms', async run(config) { await Promise.all( - config.getNodePlatforms().map(async (platform) => { + config.getTargetPlatforms().map(async (platform) => { const { downloadPath, extractDir } = getNodeDownloadInfo(config, platform); if (platform.isWindows()) { // windows executable is not extractable, it's just an .exe file diff --git a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.ts b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.ts index d942eb511ff5..d873201c12f1 100644 --- a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.ts @@ -36,7 +36,7 @@ export const VerifyExistingNodeBuilds: GlobalTask = { const shasums = await getNodeShasums(log, config.getNodeVersion()); await Promise.all( - config.getNodePlatforms().map(async (platform) => { + config.getTargetPlatforms().map(async (platform) => { const { downloadPath, downloadName } = getNodeDownloadInfo(config, platform); const sha256 = await getFileHash(downloadPath, 'sha256'); diff --git a/src/dev/build/tasks/notice_file_task.ts b/src/dev/build/tasks/notice_file_task.ts index d9196d8d6f44..75602776ff4e 100644 --- a/src/dev/build/tasks/notice_file_task.ts +++ b/src/dev/build/tasks/notice_file_task.ts @@ -53,9 +53,15 @@ export const CreateNoticeFile: Task = { }); log.info('Generating build notice'); + const { extractDir: nodeDir, version: nodeVersion } = getNodeDownloadInfo( config, - config.getPlatform('linux', 'x64') + config.hasSpecifiedPlatform() + ? config.getPlatform( + config.getTargetPlatforms()[0].getName(), + config.getTargetPlatforms()[0].getArchitecture() + ) + : config.getPlatform('linux', 'x64') ); const notice = await generateBuildNoticeText({