From f58da34c5b37bc117038cfc54822b1e9a3f37766 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Mon, 14 Jun 2021 17:12:24 +0200 Subject: [PATCH 01/20] feat: divide and merge chart values to different value files. --- .vscode/launch.json | 12 +++++++ package.json | 4 ++- src/tasks/otomi/otomi-chart.ts | 57 ++++++++++++++++++++++++++++++++++ src/tasks/otomi/settings.yaml | 0 4 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 src/tasks/otomi/otomi-chart.ts create mode 100644 src/tasks/otomi/settings.yaml diff --git a/.vscode/launch.json b/.vscode/launch.json index fa53f995..99aca978 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -112,6 +112,18 @@ }, "console": "integratedTerminal" }, + { + "type": "node", + "request": "launch", + "name": "Debug otomi-chart task", + "runtimeExecutable": "npm", + "runtimeArgs": ["run-script", "tasks:otomi-chart-dev"], + "cwd": "${workspaceRoot}", + "env": { + "DEBUG": "*" + }, + "console": "integratedTerminal" + }, { "type": "node", "request": "launch", diff --git a/package.json b/package.json index 4360ca4f..65371f3d 100644 --- a/package.json +++ b/package.json @@ -125,6 +125,8 @@ "tasks:gitea-drone-auth-dev": "ts-node-dev ./src/tasks/gitea/gitea-drone-oauth.ts", "tasks:harbor": "node dist/tasks/harbor/harbor.js", "tasks:harbor-dev": "ts-node-dev ./src/tasks/harbor/harbor.ts", + "tasks:otomi-chart": "node dist/tasks/otomi/otomi-chart.js", + "tasks:otomi-chart-dev": "ts-node-dev ./src/tasks/otomi/otomi-chart.ts", "tasks:certs-aws": "node dist/tasks/otomi/certs-aws.js", "tasks:certs-aws-dev": "ts-node-dev ./src/tasks/otomi/certs-aws.ts", "test": "NODE_ENV=test mocha --timeout 25000 -r ts-node/register -r ts-custom-error src/**/*.test.* --exit" @@ -135,4 +137,4 @@ } }, "version": "0.2.6" -} +} \ No newline at end of file diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts new file mode 100644 index 00000000..a25e06aa --- /dev/null +++ b/src/tasks/otomi/otomi-chart.ts @@ -0,0 +1,57 @@ +import { omit, merge } from 'lodash' + +const fs = require('fs') +const yaml = require('js-yaml') + +const destinationPath = '/Users/mojtaba/opt/bootstrapfiles/env' +const sourcePath = '/Users/mojtaba/repo/github/redkubes/otomi-core/chart' + +try { + const values = yaml.safeLoad(fs.readFileSync(`${sourcePath}/values.yaml`, 'utf8')) + + const settings = omit(values, ['charts', 'cluster', 'policies', 'teamConfig']) + const { charts } = values + const { cluster } = values + const { policies } = values + const { teamConfig } = values + + const settingsPath = `${destinationPath}/settings.yaml` + const clusterPath = `${destinationPath}/cluster.yaml` + const teamPath = `${destinationPath}/teams.yaml` + const policiesPath = `${destinationPath}/policies.yaml` + + const bsSettings = yaml.safeLoad(fs.readFileSync(settingsPath, 'utf8')) + const bsCluster = yaml.safeLoad(fs.readFileSync(clusterPath, 'utf8')) + const bsTeams = yaml.safeLoad(fs.readFileSync(teamPath, 'utf8')) + const bsPolicies = yaml.safeLoad(fs.readFileSync(policiesPath, 'utf8')) + + merge(bsSettings, settings) + merge(bsCluster, cluster) + merge(bsTeams, teamConfig) + merge(bsPolicies, policies) + + fs.writeFile(settingsPath, yaml.safeDump(bsSettings), (err) => { + if (err) { + console.log(err) + } + }) + fs.writeFile(clusterPath, yaml.safeDump(bsCluster), (err) => { + if (err) { + console.log(err) + } + }) + fs.writeFile(teamPath, yaml.safeDump(bsTeams), (err) => { + if (err) { + console.log(err) + } + }) + fs.writeFile(policiesPath, yaml.safeDump(bsPolicies), (err) => { + if (err) { + console.log(err) + } + }) + + console.log('done') +} catch (e) { + console.log(e) +} diff --git a/src/tasks/otomi/settings.yaml b/src/tasks/otomi/settings.yaml new file mode 100644 index 00000000..e69de29b From 4f0624579bc8daeaad1ab4d52ad67e1de569efab Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Tue, 15 Jun 2021 15:32:36 +0200 Subject: [PATCH 02/20] feat: merging all chart values in charts folder --- .vscode/launch.json | 1 + src/tasks/otomi/otomi-chart.ts | 89 ++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 99aca978..f0827d04 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,7 @@ { "version": "0.2.0", "configurations": [ + { "type": "node", "request": "launch", diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index a25e06aa..c8a360c3 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -1,54 +1,59 @@ import { omit, merge } from 'lodash' - -const fs = require('fs') -const yaml = require('js-yaml') +import yaml from 'js-yaml' +import fs from 'fs' const destinationPath = '/Users/mojtaba/opt/bootstrapfiles/env' const sourcePath = '/Users/mojtaba/repo/github/redkubes/otomi-core/chart' +function mergeValues(cat: string, valueObject, folder: string) { + const bsPath = `${folder}/${cat}.yaml` + let bsValues = yaml.safeLoad(fs.readFileSync(bsPath), 'utf8') + if (!bsValues) { + bsValues = {} + } + merge(bsValues, valueObject) + if (bsValues) { + fs.writeFile(bsPath, yaml.safeDump(bsValues), (err) => { + if (err) { + console.log(err) + } else { + console.log(`The file ${bsPath} was saved`) + } + }) + } +} + try { const values = yaml.safeLoad(fs.readFileSync(`${sourcePath}/values.yaml`, 'utf8')) - const settings = omit(values, ['charts', 'cluster', 'policies', 'teamConfig']) - const { charts } = values - const { cluster } = values - const { policies } = values - const { teamConfig } = values - - const settingsPath = `${destinationPath}/settings.yaml` - const clusterPath = `${destinationPath}/cluster.yaml` - const teamPath = `${destinationPath}/teams.yaml` - const policiesPath = `${destinationPath}/policies.yaml` - - const bsSettings = yaml.safeLoad(fs.readFileSync(settingsPath, 'utf8')) - const bsCluster = yaml.safeLoad(fs.readFileSync(clusterPath, 'utf8')) - const bsTeams = yaml.safeLoad(fs.readFileSync(teamPath, 'utf8')) - const bsPolicies = yaml.safeLoad(fs.readFileSync(policiesPath, 'utf8')) - - merge(bsSettings, settings) - merge(bsCluster, cluster) - merge(bsTeams, teamConfig) - merge(bsPolicies, policies) - - fs.writeFile(settingsPath, yaml.safeDump(bsSettings), (err) => { - if (err) { - console.log(err) - } - }) - fs.writeFile(clusterPath, yaml.safeDump(bsCluster), (err) => { - if (err) { - console.log(err) - } - }) - fs.writeFile(teamPath, yaml.safeDump(bsTeams), (err) => { - if (err) { - console.log(err) - } - }) - fs.writeFile(policiesPath, yaml.safeDump(bsPolicies), (err) => { - if (err) { - console.log(err) + mergeValues('cluster', { cluster: values.cluster }, destinationPath) + mergeValues('policies', { policies: values.policies }, destinationPath) + mergeValues('teams', { teamConfig: values.teamConfig }, destinationPath) + + const settings = omit(values, ['cluster', 'policies', 'teamConfig', 'charts']) + mergeValues('settings', settings, destinationPath) + + const charts = [ + 'cert-manager', + 'drone', + 'external-dns', + 'gatekeeper-operator', + 'gitea', + 'keycloak', + 'loki', + 'otomi-api', + 'sitespeed', + 'vault', + 'weave-scope', + ] + + charts.forEach((chart) => { + const valueObject = { + charts: { + [chart]: values.charts[chart], + }, } + mergeValues(chart, valueObject, `${destinationPath}/charts`) }) console.log('done') From b555c7f99156025a72dda725922392a93a7c596c Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Wed, 16 Jun 2021 17:49:55 +0200 Subject: [PATCH 03/20] feat: adding otomi-chart.ts to map values.yaml to different value files --- src/tasks/otomi/otomi-chart.ts | 129 ++++++++++++++++++++------------- 1 file changed, 80 insertions(+), 49 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index c8a360c3..6b789abd 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -1,62 +1,93 @@ -import { omit, merge } from 'lodash' -import yaml from 'js-yaml' +import { omit, merge, pick } from 'lodash' +import yaml, { Schema } from 'js-yaml' import fs from 'fs' +import $RefParser from '@apidevtools/json-schema-ref-parser' const destinationPath = '/Users/mojtaba/opt/bootstrapfiles/env' const sourcePath = '/Users/mojtaba/repo/github/redkubes/otomi-core/chart' +const schemaPath = '/Users/mojtaba/repo/github/redkubes/otomi-core/values-schema.yaml' +const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] -function mergeValues(cat: string, valueObject, folder: string) { - const bsPath = `${folder}/${cat}.yaml` - let bsValues = yaml.safeLoad(fs.readFileSync(bsPath), 'utf8') - if (!bsValues) { - bsValues = {} - } - merge(bsValues, valueObject) - if (bsValues) { - fs.writeFile(bsPath, yaml.safeDump(bsValues), (err) => { - if (err) { - console.log(err) +function extractSecrets(values: any, parentKey: string): any { + return Object.keys(values) + .flatMap((key) => { + const childObj = values[key] + + if (typeof childObj !== 'object') return false + if ('x-secret' in childObj) { + return `${parentKey}.${key}` + } + let address + if (schemaKeywords.includes(key)) { + address = parentKey } else { - console.log(`The file ${bsPath} was saved`) + address = `${parentKey}.${key}` } + return extractSecrets(childObj, address) }) - } + .filter(Boolean) } +function mergeValues(cat: string, valueObject, folder: string): void { + try { + const bsPath = `${folder}/${cat}.yaml` -try { - const values = yaml.safeLoad(fs.readFileSync(`${sourcePath}/values.yaml`, 'utf8')) - - mergeValues('cluster', { cluster: values.cluster }, destinationPath) - mergeValues('policies', { policies: values.policies }, destinationPath) - mergeValues('teams', { teamConfig: values.teamConfig }, destinationPath) - - const settings = omit(values, ['cluster', 'policies', 'teamConfig', 'charts']) - mergeValues('settings', settings, destinationPath) - - const charts = [ - 'cert-manager', - 'drone', - 'external-dns', - 'gatekeeper-operator', - 'gitea', - 'keycloak', - 'loki', - 'otomi-api', - 'sitespeed', - 'vault', - 'weave-scope', - ] - - charts.forEach((chart) => { - const valueObject = { - charts: { - [chart]: values.charts[chart], - }, + let bsValues + if (fs.existsSync(bsPath)) { + bsValues = yaml.load(fs.readFileSync(bsPath).toString()) + } + if (!bsValues) { + bsValues = yaml.safeLoad('{}') } - mergeValues(chart, valueObject, `${destinationPath}/charts`) - }) - console.log('done') -} catch (e) { - console.log(e) + merge(bsValues, valueObject) + fs.writeFileSync(bsPath, yaml.safeDump(bsValues)) + } catch (e) { + console.log(e) + } +} + +async function main(): Promise { + try { + const values = yaml.safeLoad(fs.readFileSync(`${sourcePath}/values.yaml`, 'utf8')) as any + + // creating secret files + const schema = yaml.safeLoad(fs.readFileSync(schemaPath, 'utf8')) as any + const derefSchema = await $RefParser.dereference(schema) + const cleanSchema = omit(derefSchema, 'definitions') + const secretsAddress = extractSecrets(cleanSchema, 'root').map((str) => str.replace('root.', '')) + const secrets = pick(values, secretsAddress) + mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) + const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) + mergeValues('secrets.settings', secretSettings, destinationPath) + Object.keys(secrets.charts).forEach((chart) => { + const valueObject = { + charts: { + [chart]: values.charts[chart], + }, + } + mergeValues(`secrets.${chart}`, valueObject, `${destinationPath}/charts`) + }) + + // creating non secret files + mergeValues('cluster', { cluster: values.cluster }, destinationPath) + mergeValues('policies', { policies: values.policies }, destinationPath) + mergeValues('teams', { teamConfig: values.teamConfig }, destinationPath) + + const settings = omit(values, ['cluster', 'policies', 'teamConfig', 'charts']) + mergeValues('settings', settings, destinationPath) + + Object.keys(values.charts).forEach((chart) => { + const valueObject = { + charts: { + [chart]: values.charts[chart], + }, + } + mergeValues(chart, valueObject, `${destinationPath}/charts`) + }) + + console.log('done') + } catch (e) { + console.log(e) + } } +main() From 91d07e3f5f652ef9a89f3af464e344aa1a54ff7a Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 17 Jun 2021 09:57:37 +0200 Subject: [PATCH 04/20] fix: removing schema index from the json path --- src/tasks/otomi/otomi-chart.ts | 50 ++++++++++++++++------------------ 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 6b789abd..8fab9665 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -1,32 +1,27 @@ -import { omit, merge, pick } from 'lodash' -import yaml, { Schema } from 'js-yaml' +import { omit, merge, pick, isNumber } from 'lodash' +import yaml from 'js-yaml' import fs from 'fs' import $RefParser from '@apidevtools/json-schema-ref-parser' const destinationPath = '/Users/mojtaba/opt/bootstrapfiles/env' const sourcePath = '/Users/mojtaba/repo/github/redkubes/otomi-core/chart' const schemaPath = '/Users/mojtaba/repo/github/redkubes/otomi-core/values-schema.yaml' + const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] -function extractSecrets(values: any, parentKey: string): any { - return Object.keys(values) +function extractSecrets(schema: any, parentKey: string): any { + return Object.keys(schema) .flatMap((key) => { - const childObj = values[key] - + const childObj = schema[key] if (typeof childObj !== 'object') return false - if ('x-secret' in childObj) { - return `${parentKey}.${key}` - } - let address - if (schemaKeywords.includes(key)) { - address = parentKey - } else { - address = `${parentKey}.${key}` - } + if ('x-secret' in childObj) return `${parentKey}.${key}` + if (key in ['anyOf', 'allOf', 'oneOf']) return extractSecrets(childObj, parentKey) + const address = schemaKeywords.includes(key) ? parentKey : `${parentKey}.${key}` return extractSecrets(childObj, address) }) .filter(Boolean) } + function mergeValues(cat: string, valueObject, folder: string): void { try { const bsPath = `${folder}/${cat}.yaml` @@ -36,7 +31,7 @@ function mergeValues(cat: string, valueObject, folder: string): void { bsValues = yaml.load(fs.readFileSync(bsPath).toString()) } if (!bsValues) { - bsValues = yaml.safeLoad('{}') + bsValues = {} } merge(bsValues, valueObject) @@ -53,10 +48,12 @@ async function main(): Promise { // creating secret files const schema = yaml.safeLoad(fs.readFileSync(schemaPath, 'utf8')) as any const derefSchema = await $RefParser.dereference(schema) - const cleanSchema = omit(derefSchema, 'definitions') - const secretsAddress = extractSecrets(cleanSchema, 'root').map((str) => str.replace('root.', '')) - const secrets = pick(values, secretsAddress) - mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) + const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later + const secretsJsonPath = extractSecrets(cleanSchema, 'root').map((str) => str.replace('root.', '')) + console.log(secretsJsonPath) + const secrets = pick(values, secretsJsonPath) + console.log(secrets) + // mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) // FIXME: lets fix the team part later const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) mergeValues('secrets.settings', secretSettings, destinationPath) Object.keys(secrets.charts).forEach((chart) => { @@ -67,19 +64,20 @@ async function main(): Promise { } mergeValues(`secrets.${chart}`, valueObject, `${destinationPath}/charts`) }) + const plainValues = omit(values, secretsJsonPath) as any // creating non secret files - mergeValues('cluster', { cluster: values.cluster }, destinationPath) - mergeValues('policies', { policies: values.policies }, destinationPath) - mergeValues('teams', { teamConfig: values.teamConfig }, destinationPath) + mergeValues('cluster', { cluster: plainValues.cluster }, destinationPath) + mergeValues('policies', { policies: plainValues.policies }, destinationPath) + mergeValues('teams', { teamConfig: plainValues.teamConfig }, destinationPath) - const settings = omit(values, ['cluster', 'policies', 'teamConfig', 'charts']) + const settings = omit(plainValues, ['cluster', 'policies', 'teamConfig', 'charts']) mergeValues('settings', settings, destinationPath) - Object.keys(values.charts).forEach((chart) => { + Object.keys(plainValues.charts).forEach((chart) => { const valueObject = { charts: { - [chart]: values.charts[chart], + [chart]: plainValues.charts[chart], }, } mergeValues(chart, valueObject, `${destinationPath}/charts`) From dc2069fff4c2df94b19543b45f6c73d804730d5d Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 17 Jun 2021 13:13:08 +0200 Subject: [PATCH 05/20] fix: adding env variables for source and target and schema --- .env.sample | 7 ++++++- src/tasks/otomi/otomi-chart.ts | 30 +++++++++++++++++------------- src/validators.ts | 3 +++ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.env.sample b/.env.sample index 05f04631..c253b115 100644 --- a/.env.sample +++ b/.env.sample @@ -36,4 +36,9 @@ GITEA_URL='http://127.0.0.1:8082' DRONE_URL='https://drone.dev.gke.otomi.cloud' # Drone task -DRONE_TOKEN='' \ No newline at end of file +DRONE_TOKEN='' + +# otomi chart +OTOMI_VALUES_INPUT = '/cache/values.yaml' +OTOMI_SCHEMA_PATH = '/cache/values-schema.yaml' +OTOMI_VALUES_TARGET = '/env' diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 8fab9665..1e215e32 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -1,11 +1,15 @@ -import { omit, merge, pick, isNumber } from 'lodash' +import { omit, merge, pick } from 'lodash' import yaml from 'js-yaml' import fs from 'fs' import $RefParser from '@apidevtools/json-schema-ref-parser' -const destinationPath = '/Users/mojtaba/opt/bootstrapfiles/env' -const sourcePath = '/Users/mojtaba/repo/github/redkubes/otomi-core/chart' -const schemaPath = '/Users/mojtaba/repo/github/redkubes/otomi-core/values-schema.yaml' +import { cleanEnv, OTOMI_VALUES_INPUT, OTOMI_SCHEMA_PATH, OTOMI_VALUES_TARGET } from '../../validators' + +const env = cleanEnv({ + OTOMI_VALUES_INPUT, + OTOMI_SCHEMA_PATH, + OTOMI_VALUES_TARGET, +}) const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] @@ -43,10 +47,10 @@ function mergeValues(cat: string, valueObject, folder: string): void { async function main(): Promise { try { - const values = yaml.safeLoad(fs.readFileSync(`${sourcePath}/values.yaml`, 'utf8')) as any + const values = yaml.safeLoad(fs.readFileSync(env.OTOMI_VALUES_INPUT, 'utf8')) as any // creating secret files - const schema = yaml.safeLoad(fs.readFileSync(schemaPath, 'utf8')) as any + const schema = yaml.safeLoad(fs.readFileSync(env.OTOMI_SCHEMA_PATH, 'utf8')) as any const derefSchema = await $RefParser.dereference(schema) const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later const secretsJsonPath = extractSecrets(cleanSchema, 'root').map((str) => str.replace('root.', '')) @@ -55,24 +59,24 @@ async function main(): Promise { console.log(secrets) // mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) // FIXME: lets fix the team part later const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) - mergeValues('secrets.settings', secretSettings, destinationPath) + mergeValues('secrets.settings', secretSettings, env.OTOMI_VALUES_TARGET) Object.keys(secrets.charts).forEach((chart) => { const valueObject = { charts: { [chart]: values.charts[chart], }, } - mergeValues(`secrets.${chart}`, valueObject, `${destinationPath}/charts`) + mergeValues(`secrets.${chart}`, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) }) const plainValues = omit(values, secretsJsonPath) as any // creating non secret files - mergeValues('cluster', { cluster: plainValues.cluster }, destinationPath) - mergeValues('policies', { policies: plainValues.policies }, destinationPath) - mergeValues('teams', { teamConfig: plainValues.teamConfig }, destinationPath) + mergeValues('cluster', { cluster: plainValues.cluster }, env.OTOMI_VALUES_TARGET) + mergeValues('policies', { policies: plainValues.policies }, env.OTOMI_VALUES_TARGET) + mergeValues('teams', { teamConfig: plainValues.teamConfig }, env.OTOMI_VALUES_TARGET) const settings = omit(plainValues, ['cluster', 'policies', 'teamConfig', 'charts']) - mergeValues('settings', settings, destinationPath) + mergeValues('settings', settings, env.OTOMI_VALUES_TARGET) Object.keys(plainValues.charts).forEach((chart) => { const valueObject = { @@ -80,7 +84,7 @@ async function main(): Promise { [chart]: plainValues.charts[chart], }, } - mergeValues(chart, valueObject, `${destinationPath}/charts`) + mergeValues(chart, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) }) console.log('done') diff --git a/src/validators.ts b/src/validators.ts index 3f12536a..ba48533a 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -41,6 +41,9 @@ export const TEAM_NAMES = json({ desc: 'A list of team names in JSON format' }) export const TENANT_CLIENT_ID = str({ desc: 'The tenant client id' }) export const TENANT_CLIENT_SECRET = str({ desc: 'The tenant client secret' }) export const TENANT_ID = str({ desc: 'The tenant ID' }) +export const OTOMI_VALUES_INPUT = str({ desc: 'values.yaml file for the chart values' }) +export const OTOMI_SCHEMA_PATH = str({ desc: 'The path for the otomi value schema' }) +export const OTOMI_VALUES_TARGET = str({ desc: 'The path of the otomi value folder' }) const { env } = process export function cleanEnv( From 2963fbc8cbc24e8da2bfebf50fb8ca7ccae49be7 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 17 Jun 2021 13:17:49 +0200 Subject: [PATCH 06/20] fix: removing console.logs --- src/tasks/otomi/otomi-chart.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 1e215e32..43da449a 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -54,9 +54,7 @@ async function main(): Promise { const derefSchema = await $RefParser.dereference(schema) const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later const secretsJsonPath = extractSecrets(cleanSchema, 'root').map((str) => str.replace('root.', '')) - console.log(secretsJsonPath) const secrets = pick(values, secretsJsonPath) - console.log(secrets) // mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) // FIXME: lets fix the team part later const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) mergeValues('secrets.settings', secretSettings, env.OTOMI_VALUES_TARGET) @@ -68,6 +66,8 @@ async function main(): Promise { } mergeValues(`secrets.${chart}`, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) }) + + // removing secrets const plainValues = omit(values, secretsJsonPath) as any // creating non secret files @@ -87,7 +87,7 @@ async function main(): Promise { mergeValues(chart, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) }) - console.log('done') + console.log('Done.') } catch (e) { console.log(e) } From 2467d68a50f2d90fe89be7daf70185f683a6988d Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 17 Jun 2021 16:05:08 +0200 Subject: [PATCH 07/20] fix: unit test added --- src/tasks/otomi/otomi-chart.test.ts | 57 ++++++++++++++++ src/tasks/otomi/otomi-chart.ts | 100 +++++++++++++--------------- 2 files changed, 103 insertions(+), 54 deletions(-) create mode 100644 src/tasks/otomi/otomi-chart.test.ts diff --git a/src/tasks/otomi/otomi-chart.test.ts b/src/tasks/otomi/otomi-chart.test.ts new file mode 100644 index 00000000..ad2a674b --- /dev/null +++ b/src/tasks/otomi/otomi-chart.test.ts @@ -0,0 +1,57 @@ +import { expect } from 'chai' +import extractSecrets from './otomi-chart' + +describe('simple schema', () => { + it('extract secrets json paths from schema', () => { + const schema = { + $schema: 'http://json-schema.org/draft-07/schema', + additionalProperties: false, + definitions: { + rawValues: { description: 'bla bla', type: 'object' }, + }, + properties: { + apps: { description: 'console.', type: 'object' }, + pullSecret: { + description: 'console.', + type: 'string', + [`x-secret`]: 'true', + }, + azure: { + hello: { + description: 'console.', + type: 'string', + [`x-secret`]: 'true', + }, + }, + }, + } + const expectedResult = ['root.pullSecret', 'root.azure.hello'] + const x = extractSecrets(schema, 'root') + expect(x).to.deep.equal(expectedResult) + }) +}) + +describe('oneOf', () => { + it('extract secrets json paths from schema', () => { + const schema = { + oneOf: [ + { aws: { password: { type: 'string', [`x-secret`]: 'true' } } }, + { google: { password: { type: 'string', [`x-secret`]: 'true' } } }, + { azure: { password: { type: 'string', [`x-secret`]: 'true' } } }, + ], + } + + const expectedResult = ['root.aws.password', 'root.google.password', 'root.azure.password'] + const x = extractSecrets(schema, 'root') + expect(x).to.deep.equal(expectedResult) + }) +}) + +describe('empty schema', () => { + it('a simple schema', () => { + const schema = {} + const expectedResult = [] + const x = extractSecrets(schema, '') + expect(x).to.deep.equal(expectedResult) + }) +}) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 43da449a..503f75a3 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -13,7 +13,7 @@ const env = cleanEnv({ const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] -function extractSecrets(schema: any, parentKey: string): any { +export default function extractSecrets(schema: any, parentKey: string): any { return Object.keys(schema) .flatMap((key) => { const childObj = schema[key] @@ -27,69 +27,61 @@ function extractSecrets(schema: any, parentKey: string): any { } function mergeValues(cat: string, valueObject, folder: string): void { - try { - const bsPath = `${folder}/${cat}.yaml` + const bsPath = `${folder}/${cat}.yaml` - let bsValues - if (fs.existsSync(bsPath)) { - bsValues = yaml.load(fs.readFileSync(bsPath).toString()) - } - if (!bsValues) { - bsValues = {} - } - - merge(bsValues, valueObject) - fs.writeFileSync(bsPath, yaml.safeDump(bsValues)) - } catch (e) { - console.log(e) + let bsValues + if (fs.existsSync(bsPath)) { + bsValues = yaml.load(fs.readFileSync(bsPath).toString()) } + if (!bsValues) { + bsValues = {} + } + + merge(bsValues, valueObject) + fs.writeFileSync(bsPath, yaml.safeDump(bsValues)) } async function main(): Promise { - try { - const values = yaml.safeLoad(fs.readFileSync(env.OTOMI_VALUES_INPUT, 'utf8')) as any + const values = yaml.safeLoad(fs.readFileSync(env.OTOMI_VALUES_INPUT, 'utf8')) as any - // creating secret files - const schema = yaml.safeLoad(fs.readFileSync(env.OTOMI_SCHEMA_PATH, 'utf8')) as any - const derefSchema = await $RefParser.dereference(schema) - const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later - const secretsJsonPath = extractSecrets(cleanSchema, 'root').map((str) => str.replace('root.', '')) - const secrets = pick(values, secretsJsonPath) - // mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) // FIXME: lets fix the team part later - const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) - mergeValues('secrets.settings', secretSettings, env.OTOMI_VALUES_TARGET) - Object.keys(secrets.charts).forEach((chart) => { - const valueObject = { - charts: { - [chart]: values.charts[chart], - }, - } - mergeValues(`secrets.${chart}`, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) - }) + // creating secret files + const schema = yaml.safeLoad(fs.readFileSync(env.OTOMI_SCHEMA_PATH, 'utf8')) as any + const derefSchema = await $RefParser.dereference(schema) + const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later + const secretsJsonPath = extractSecrets(cleanSchema, 'root').map((str) => str.replace('root.', '')) + const secrets = pick(values, secretsJsonPath) + // mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) // FIXME: lets fix the team part later + const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) + mergeValues('secrets.settings', secretSettings, env.OTOMI_VALUES_TARGET) + Object.keys(secrets.charts).map((chart) => { + const valueObject = { + charts: { + [chart]: values.charts[chart], + }, + } + mergeValues(`secrets.${chart}`, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) + }) - // removing secrets - const plainValues = omit(values, secretsJsonPath) as any + // removing secrets + const plainValues = omit(values, secretsJsonPath) as any - // creating non secret files - mergeValues('cluster', { cluster: plainValues.cluster }, env.OTOMI_VALUES_TARGET) - mergeValues('policies', { policies: plainValues.policies }, env.OTOMI_VALUES_TARGET) - mergeValues('teams', { teamConfig: plainValues.teamConfig }, env.OTOMI_VALUES_TARGET) + // creating non secret files + mergeValues('cluster', { cluster: plainValues.cluster }, env.OTOMI_VALUES_TARGET) + mergeValues('policies', { policies: plainValues.policies }, env.OTOMI_VALUES_TARGET) + mergeValues('teams', { teamConfig: plainValues.teamConfig }, env.OTOMI_VALUES_TARGET) - const settings = omit(plainValues, ['cluster', 'policies', 'teamConfig', 'charts']) - mergeValues('settings', settings, env.OTOMI_VALUES_TARGET) + const settings = omit(plainValues, ['cluster', 'policies', 'teamConfig', 'charts']) + mergeValues('settings', settings, env.OTOMI_VALUES_TARGET) - Object.keys(plainValues.charts).forEach((chart) => { - const valueObject = { - charts: { - [chart]: plainValues.charts[chart], - }, - } - mergeValues(chart, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) - }) + Object.keys(plainValues.charts).map((chart) => { + const valueObject = { + charts: { + [chart]: plainValues.charts[chart], + }, + } + mergeValues(chart, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) + }) - console.log('Done.') - } catch (e) { - console.log(e) - } + console.log('otomi chart values merged with the bootstrapped values.') } main() From e0aebce07c13db3ff8d6e7073e8ccce92aabdcd7 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 17 Jun 2021 16:22:55 +0200 Subject: [PATCH 08/20] fix: extractSecrets output type fixed. --- src/tasks/otomi/otomi-chart.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 503f75a3..8807b36b 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -13,7 +13,7 @@ const env = cleanEnv({ const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] -export default function extractSecrets(schema: any, parentKey: string): any { +export default function extractSecrets(schema: any, parentKey: string): Array { return Object.keys(schema) .flatMap((key) => { const childObj = schema[key] @@ -23,7 +23,7 @@ export default function extractSecrets(schema: any, parentKey: string): any { const address = schemaKeywords.includes(key) ? parentKey : `${parentKey}.${key}` return extractSecrets(childObj, address) }) - .filter(Boolean) + .filter(Boolean) as Array } function mergeValues(cat: string, valueObject, folder: string): void { From 7401ca16bf929af4850f7637d64737be976ab4bf Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 17 Jun 2021 17:14:02 +0200 Subject: [PATCH 09/20] fix: signature of mergeValues function changed --- src/tasks/otomi/otomi-chart.ts | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 8807b36b..0862b1f6 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -26,19 +26,16 @@ export default function extractSecrets(schema: any, parentKey: string): Array } -function mergeValues(cat: string, valueObject, folder: string): void { - const bsPath = `${folder}/${cat}.yaml` - +function mergeValues(targetPath: string, valueObject): void { let bsValues - if (fs.existsSync(bsPath)) { - bsValues = yaml.load(fs.readFileSync(bsPath).toString()) + if (fs.existsSync(targetPath)) { + bsValues = yaml.load(fs.readFileSync(targetPath).toString()) } if (!bsValues) { bsValues = {} } - merge(bsValues, valueObject) - fs.writeFileSync(bsPath, yaml.safeDump(bsValues)) + fs.writeFileSync(targetPath, yaml.safeDump(bsValues)) } async function main(): Promise { @@ -52,34 +49,34 @@ async function main(): Promise { const secrets = pick(values, secretsJsonPath) // mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) // FIXME: lets fix the team part later const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) - mergeValues('secrets.settings', secretSettings, env.OTOMI_VALUES_TARGET) - Object.keys(secrets.charts).map((chart) => { + mergeValues(`${env.OTOMI_VALUES_TARGET}/secrets.settings.yaml`, secretSettings) + Object.keys(secrets.charts).forEach((chart) => { const valueObject = { charts: { [chart]: values.charts[chart], }, } - mergeValues(`secrets.${chart}`, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) + mergeValues(`${env.OTOMI_VALUES_TARGET}/charts/secrets.${chart}.yaml`, valueObject) }) // removing secrets const plainValues = omit(values, secretsJsonPath) as any // creating non secret files - mergeValues('cluster', { cluster: plainValues.cluster }, env.OTOMI_VALUES_TARGET) - mergeValues('policies', { policies: plainValues.policies }, env.OTOMI_VALUES_TARGET) - mergeValues('teams', { teamConfig: plainValues.teamConfig }, env.OTOMI_VALUES_TARGET) + mergeValues(`${env.OTOMI_VALUES_TARGET}/cluster.yaml`, { cluster: plainValues.cluster }) + mergeValues(`${env.OTOMI_VALUES_TARGET}/policies.yaml`, { policies: plainValues.policies }) + mergeValues(`${env.OTOMI_VALUES_TARGET}/teams.yaml`, { teamConfig: plainValues.teamConfig }) const settings = omit(plainValues, ['cluster', 'policies', 'teamConfig', 'charts']) - mergeValues('settings', settings, env.OTOMI_VALUES_TARGET) + mergeValues(`${env.OTOMI_VALUES_TARGET}/settings.yaml`, settings) - Object.keys(plainValues.charts).map((chart) => { + Object.keys(plainValues.charts).forEach((chart) => { const valueObject = { charts: { [chart]: plainValues.charts[chart], }, } - mergeValues(chart, valueObject, `${env.OTOMI_VALUES_TARGET}/charts`) + mergeValues(`${env.OTOMI_VALUES_TARGET}/charts/${chart}.yaml`, valueObject) }) console.log('otomi chart values merged with the bootstrapped values.') From 48577e023ccd08136cbf8cf5e203528fa03fbe54 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 17 Jun 2021 17:17:56 +0200 Subject: [PATCH 10/20] fix: removing space from the env.sample --- .env.sample | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.sample b/.env.sample index c253b115..1a79295f 100644 --- a/.env.sample +++ b/.env.sample @@ -39,6 +39,6 @@ DRONE_URL='https://drone.dev.gke.otomi.cloud' DRONE_TOKEN='' # otomi chart -OTOMI_VALUES_INPUT = '/cache/values.yaml' -OTOMI_SCHEMA_PATH = '/cache/values-schema.yaml' -OTOMI_VALUES_TARGET = '/env' +OTOMI_VALUES_INPUT='/cache/values.yaml' +OTOMI_SCHEMA_PATH='/cache/values-schema.yaml' +OTOMI_VALUES_TARGET='/env' From 26ab0aaba77f7435bdb1c2d51ab593bae6c1a8d6 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 17 Jun 2021 17:43:05 +0200 Subject: [PATCH 11/20] fix: fixing the test functions --- src/tasks/otomi/otomi-chart.test.ts | 43 +++++++++++------------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.test.ts b/src/tasks/otomi/otomi-chart.test.ts index ad2a674b..c8521614 100644 --- a/src/tasks/otomi/otomi-chart.test.ts +++ b/src/tasks/otomi/otomi-chart.test.ts @@ -1,8 +1,8 @@ import { expect } from 'chai' import extractSecrets from './otomi-chart' -describe('simple schema', () => { - it('extract secrets json paths from schema', () => { +describe('otomi-chart', () => { + it('It should extract the json path of the secrets found in the schema', () => { const schema = { $schema: 'http://json-schema.org/draft-07/schema', additionalProperties: false, @@ -16,39 +16,28 @@ describe('simple schema', () => { type: 'string', [`x-secret`]: 'true', }, - azure: { - hello: { - description: 'console.', - type: 'string', - [`x-secret`]: 'true', + provider: { + properties: { + oneOf: [ + { aws: { password: { type: 'string', [`x-secret`]: 'true' } } }, + { google: { password: { type: 'string', [`x-secret`]: 'true' } } }, + { azure: { password: { type: 'string', [`x-secret`]: 'true' } } }, + ], }, }, }, } - const expectedResult = ['root.pullSecret', 'root.azure.hello'] + const expectedResult = [ + 'root.pullSecret', + 'root.provider.aws.password', + 'root.provider.google.password', + 'root.provider.azure.password', + ] const x = extractSecrets(schema, 'root') expect(x).to.deep.equal(expectedResult) }) -}) - -describe('oneOf', () => { - it('extract secrets json paths from schema', () => { - const schema = { - oneOf: [ - { aws: { password: { type: 'string', [`x-secret`]: 'true' } } }, - { google: { password: { type: 'string', [`x-secret`]: 'true' } } }, - { azure: { password: { type: 'string', [`x-secret`]: 'true' } } }, - ], - } - - const expectedResult = ['root.aws.password', 'root.google.password', 'root.azure.password'] - const x = extractSecrets(schema, 'root') - expect(x).to.deep.equal(expectedResult) - }) -}) -describe('empty schema', () => { - it('a simple schema', () => { + it('It should return an empty array if the input schema is empty', () => { const schema = {} const expectedResult = [] const x = extractSecrets(schema, '') From 51ac0ec68f0c2dcfc680159405f614510ada8aff Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Fri, 18 Jun 2021 11:21:27 +0200 Subject: [PATCH 12/20] fix: adding optional parameter to extractSecrets function --- src/tasks/otomi/otomi-chart.test.ts | 10 +++++----- src/tasks/otomi/otomi-chart.ts | 12 +++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.test.ts b/src/tasks/otomi/otomi-chart.test.ts index c8521614..1cf2413e 100644 --- a/src/tasks/otomi/otomi-chart.test.ts +++ b/src/tasks/otomi/otomi-chart.test.ts @@ -28,12 +28,12 @@ describe('otomi-chart', () => { }, } const expectedResult = [ - 'root.pullSecret', - 'root.provider.aws.password', - 'root.provider.google.password', - 'root.provider.azure.password', + 'pullSecret', + 'provider.aws.password', + 'provider.google.password', + 'provider.azure.password', ] - const x = extractSecrets(schema, 'root') + const x = extractSecrets(schema) expect(x).to.deep.equal(expectedResult) }) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 0862b1f6..1fb7a12d 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -13,14 +13,16 @@ const env = cleanEnv({ const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] -export default function extractSecrets(schema: any, parentKey: string): Array { +export default function extractSecrets(schema: any, parentAddress?: string): Array { return Object.keys(schema) .flatMap((key) => { const childObj = schema[key] if (typeof childObj !== 'object') return false - if ('x-secret' in childObj) return `${parentKey}.${key}` - if (key in ['anyOf', 'allOf', 'oneOf']) return extractSecrets(childObj, parentKey) - const address = schemaKeywords.includes(key) ? parentKey : `${parentKey}.${key}` + if ('x-secret' in childObj) return parentAddress ? `${parentAddress}.${key}` : key + let address + if (schemaKeywords.includes(key) || !isNaN(Number(key))) address = parentAddress + else if (parentAddress === undefined) address = key + else address = `${parentAddress}.${key}` return extractSecrets(childObj, address) }) .filter(Boolean) as Array @@ -45,7 +47,7 @@ async function main(): Promise { const schema = yaml.safeLoad(fs.readFileSync(env.OTOMI_SCHEMA_PATH, 'utf8')) as any const derefSchema = await $RefParser.dereference(schema) const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later - const secretsJsonPath = extractSecrets(cleanSchema, 'root').map((str) => str.replace('root.', '')) + const secretsJsonPath = extractSecrets(cleanSchema) const secrets = pick(values, secretsJsonPath) // mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) // FIXME: lets fix the team part later const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) From e716b589b7a2e13365f9dbc813fc5c64d078a880 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Fri, 18 Jun 2021 16:53:06 +0200 Subject: [PATCH 13/20] fix: fixing the json sample in test function and export main() --- src/tasks/otomi/otomi-chart.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.test.ts b/src/tasks/otomi/otomi-chart.test.ts index 1cf2413e..fb5487d4 100644 --- a/src/tasks/otomi/otomi-chart.test.ts +++ b/src/tasks/otomi/otomi-chart.test.ts @@ -14,14 +14,14 @@ describe('otomi-chart', () => { pullSecret: { description: 'console.', type: 'string', - [`x-secret`]: 'true', + 'x-secret': 'true', }, provider: { properties: { oneOf: [ - { aws: { password: { type: 'string', [`x-secret`]: 'true' } } }, - { google: { password: { type: 'string', [`x-secret`]: 'true' } } }, - { azure: { password: { type: 'string', [`x-secret`]: 'true' } } }, + { aws: { password: { type: 'string', 'x-secret': 'true' } } }, + { google: { password: { type: 'string', 'x-secret': 'true' } } }, + { azure: { password: { type: 'string', 'x-secret': 'true' } } }, ], }, }, From 4cb57731f622471b35a661d3eebd069380cc06c5 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Fri, 18 Jun 2021 16:53:45 +0200 Subject: [PATCH 14/20] fix: fixing function signatures --- src/tasks/otomi/otomi-chart.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 1fb7a12d..31baf9e8 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -13,7 +13,7 @@ const env = cleanEnv({ const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] -export default function extractSecrets(schema: any, parentAddress?: string): Array { +export function extractSecrets(schema: any, parentAddress?: string): Array { return Object.keys(schema) .flatMap((key) => { const childObj = schema[key] @@ -40,7 +40,7 @@ function mergeValues(targetPath: string, valueObject): void { fs.writeFileSync(targetPath, yaml.safeDump(bsValues)) } -async function main(): Promise { +export default async function main(): Promise { const values = yaml.safeLoad(fs.readFileSync(env.OTOMI_VALUES_INPUT, 'utf8')) as any // creating secret files @@ -83,4 +83,7 @@ async function main(): Promise { console.log('otomi chart values merged with the bootstrapped values.') } -main() + +if (typeof require !== 'undefined' && require.main === module) { + main() +} From fe253dbc75a7ae27e52aae25f0bc0c619955f993 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Mon, 21 Jun 2021 10:46:37 +0200 Subject: [PATCH 15/20] fix: isNaN changed to Number.isNaN --- src/tasks/otomi/otomi-chart.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 31baf9e8..3119f91e 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -20,7 +20,7 @@ export function extractSecrets(schema: any, parentAddress?: string): Array { const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later const secretsJsonPath = extractSecrets(cleanSchema) const secrets = pick(values, secretsJsonPath) - // mergeValues('secrets.team', { teamConfig: secrets.teamConfig }, destinationPath) // FIXME: lets fix the team part later + // mergeValues(`${env.OTOMI_VALUES_TARGET}/secrets.team.yaml`, { teamConfig: secrets.teamConfig }) // FIXME: lets fix the team part later const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) mergeValues(`${env.OTOMI_VALUES_TARGET}/secrets.settings.yaml`, secretSettings) Object.keys(secrets.charts).forEach((chart) => { From 625210a072ba0119d7c8c2f0d330baf7be86e7e5 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Mon, 21 Jun 2021 11:14:32 +0200 Subject: [PATCH 16/20] fix: import statement in test file fixed. --- src/tasks/otomi/otomi-chart.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/otomi/otomi-chart.test.ts b/src/tasks/otomi/otomi-chart.test.ts index fb5487d4..89aea33a 100644 --- a/src/tasks/otomi/otomi-chart.test.ts +++ b/src/tasks/otomi/otomi-chart.test.ts @@ -1,5 +1,5 @@ import { expect } from 'chai' -import extractSecrets from './otomi-chart' +import { extractSecrets } from './otomi-chart' describe('otomi-chart', () => { it('It should extract the json path of the secrets found in the schema', () => { From d1a14fa5e15be339649fe30ad88e12484f9351e6 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Wed, 23 Jun 2021 17:37:36 +0200 Subject: [PATCH 17/20] fix env var names and check input undefined for merge function --- src/tasks/otomi/otomi-chart.ts | 23 +++++++++++++---------- src/validators.ts | 6 +++--- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 3119f91e..5b15c452 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -3,12 +3,12 @@ import yaml from 'js-yaml' import fs from 'fs' import $RefParser from '@apidevtools/json-schema-ref-parser' -import { cleanEnv, OTOMI_VALUES_INPUT, OTOMI_SCHEMA_PATH, OTOMI_VALUES_TARGET } from '../../validators' +import { cleanEnv, OTOMI_VALUES_INPUT, OTOMI_SCHEMA_PATH, OTOMI_ENV_DIR } from '../../validators' const env = cleanEnv({ OTOMI_VALUES_INPUT, OTOMI_SCHEMA_PATH, - OTOMI_VALUES_TARGET, + OTOMI_ENV_DIR, }) const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] @@ -49,28 +49,31 @@ export default async function main(): Promise { const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later const secretsJsonPath = extractSecrets(cleanSchema) const secrets = pick(values, secretsJsonPath) - // mergeValues(`${env.OTOMI_VALUES_TARGET}/secrets.team.yaml`, { teamConfig: secrets.teamConfig }) // FIXME: lets fix the team part later + console.log(secretsJsonPath) + console.log(secrets) + + // mergeValues(`${env.OTOMI_ENV_DIR}/env/secrets.team.yaml`, { teamConfig: secrets.teamConfig }) // FIXME: lets fix the team part later const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) - mergeValues(`${env.OTOMI_VALUES_TARGET}/secrets.settings.yaml`, secretSettings) + if (secretSettings) mergeValues(`${env.OTOMI_ENV_DIR}/env/secrets.settings.yaml`, secretSettings) Object.keys(secrets.charts).forEach((chart) => { const valueObject = { charts: { [chart]: values.charts[chart], }, } - mergeValues(`${env.OTOMI_VALUES_TARGET}/charts/secrets.${chart}.yaml`, valueObject) + mergeValues(`${env.OTOMI_ENV_DIR}/env/charts/secrets.${chart}.yaml`, valueObject) }) // removing secrets const plainValues = omit(values, secretsJsonPath) as any // creating non secret files - mergeValues(`${env.OTOMI_VALUES_TARGET}/cluster.yaml`, { cluster: plainValues.cluster }) - mergeValues(`${env.OTOMI_VALUES_TARGET}/policies.yaml`, { policies: plainValues.policies }) - mergeValues(`${env.OTOMI_VALUES_TARGET}/teams.yaml`, { teamConfig: plainValues.teamConfig }) + if (plainValues.cluster) mergeValues(`${env.OTOMI_ENV_DIR}/env/cluster.yaml`, { cluster: plainValues.cluster }) + if (plainValues.policies) mergeValues(`${env.OTOMI_ENV_DIR}/env/policies.yaml`, { policies: plainValues.policies }) + if (plainValues.teamConfig) mergeValues(`${env.OTOMI_ENV_DIR}/env/teams.yaml`, { teamConfig: plainValues.teamConfig }) const settings = omit(plainValues, ['cluster', 'policies', 'teamConfig', 'charts']) - mergeValues(`${env.OTOMI_VALUES_TARGET}/settings.yaml`, settings) + if (settings) mergeValues(`${env.OTOMI_ENV_DIR}/env/settings.yaml`, settings) Object.keys(plainValues.charts).forEach((chart) => { const valueObject = { @@ -78,7 +81,7 @@ export default async function main(): Promise { [chart]: plainValues.charts[chart], }, } - mergeValues(`${env.OTOMI_VALUES_TARGET}/charts/${chart}.yaml`, valueObject) + mergeValues(`${env.OTOMI_ENV_DIR}/env/charts/${chart}.yaml`, valueObject) }) console.log('otomi chart values merged with the bootstrapped values.') diff --git a/src/validators.ts b/src/validators.ts index f95936b2..afc3645b 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -41,9 +41,9 @@ export const TEAM_NAMES = json({ desc: 'A list of team names in JSON format' }) export const TENANT_CLIENT_ID = str({ desc: 'The tenant client id' }) export const TENANT_CLIENT_SECRET = str({ desc: 'The tenant client secret' }) export const TENANT_ID = str({ desc: 'The tenant ID' }) -export const OTOMI_VALUES_INPUT = str({ desc: 'values.yaml file for the chart values' }) -export const OTOMI_SCHEMA_PATH = str({ desc: 'The path for the otomi value schema' }) -export const OTOMI_VALUES_TARGET = str({ desc: 'The path of the otomi value folder' }) +export const OTOMI_VALUES_INPUT = str({ desc: 'The chart values.yaml file' }) +export const OTOMI_SCHEMA_PATH = str({ desc: 'The path to the values-schema.yaml schema file' }) +export const OTOMI_ENV_DIR = str({ desc: 'The path to the otomi-values folder' }) const { env } = process export function cleanEnv( From 0be826b31f17f09ec861fa203f766cd671926c79 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Fri, 25 Jun 2021 21:48:35 +0200 Subject: [PATCH 18/20] fix: fixing merge function with secret files --- Dockerfile | 3 +-- src/tasks/otomi/otomi-chart.ts | 32 +++++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd9f6a13..30946917 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,8 +25,7 @@ COPY . .eslintrc.yml ./ ARG CI=true ENV NODE_ENV=test -RUN npm run lint -RUN npm run test +RUN if [ "$SKIP_TESTS" = 'false' ]; then npm run lint && npm run test; fi RUN npm run build # --------------- Cleanup diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 5b15c452..00dacfd8 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -28,16 +28,30 @@ export function extractSecrets(schema: any, parentAddress?: string): Array } -function mergeValues(targetPath: string, valueObject): void { - let bsValues +function mergeValues(targetPath: string, newValues): void { + let values if (fs.existsSync(targetPath)) { - bsValues = yaml.load(fs.readFileSync(targetPath).toString()) - } - if (!bsValues) { - bsValues = {} + if (targetPath.includes('/secrets.')) { + values = yaml.load(fs.readFileSync(`${targetPath}.dec`).toString()) + } else { + values = yaml.load(fs.readFileSync(targetPath).toString()) + } + + if (!values) { + values = {} + } + + merge(values, newValues) + + if (targetPath.includes('/secrets.')) { + fs.writeFileSync(`${targetPath}.dec`, yaml.safeDump(values)) + } else { + fs.writeFileSync(targetPath, yaml.safeDump(values)) + } + } else { + // if the targetPath doesn't exist, just create it and write the valueObject on it. Doesn't matter if it is secret or not. and always write in its yaml file + fs.writeFileSync(targetPath, yaml.safeDump(newValues)) } - merge(bsValues, valueObject) - fs.writeFileSync(targetPath, yaml.safeDump(bsValues)) } export default async function main(): Promise { @@ -52,7 +66,7 @@ export default async function main(): Promise { console.log(secretsJsonPath) console.log(secrets) - // mergeValues(`${env.OTOMI_ENV_DIR}/env/secrets.team.yaml`, { teamConfig: secrets.teamConfig }) // FIXME: lets fix the team part later + // mergeValues(`${env.OTOMI_ENV_DIR}/env/secrets.teams.yaml`, { teamConfig: secrets.teamConfig }) // FIXME: lets fix the team part later const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts']) if (secretSettings) mergeValues(`${env.OTOMI_ENV_DIR}/env/secrets.settings.yaml`, secretSettings) Object.keys(secrets.charts).forEach((chart) => { From 7938d4ab29f04fee56efb15d1c8dd7a44041b2d6 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 8 Jul 2021 22:29:36 +0200 Subject: [PATCH 19/20] fix: merge logic [ci skip] --- src/tasks/otomi/otomi-chart.ts | 38 ++++++++++++++-------------------- src/utils.test.ts | 9 +++++++- src/utils.ts | 10 ++++++++- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 00dacfd8..4f452584 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -4,6 +4,7 @@ import fs from 'fs' import $RefParser from '@apidevtools/json-schema-ref-parser' import { cleanEnv, OTOMI_VALUES_INPUT, OTOMI_SCHEMA_PATH, OTOMI_ENV_DIR } from '../../validators' +import { cleanValues } from '../../utils' const env = cleanEnv({ OTOMI_VALUES_INPUT, @@ -13,6 +14,9 @@ const env = cleanEnv({ const schemaKeywords = ['properties', 'anyOf', 'allOf', 'oneOf'] +let suffix = '' +if (fs.existsSync(`${env.OTOMI_ENV_DIR}/.sops.yaml`)) suffix = '.dec' + export function extractSecrets(schema: any, parentAddress?: string): Array { return Object.keys(schema) .flatMap((key) => { @@ -28,30 +32,20 @@ export function extractSecrets(schema: any, parentAddress?: string): Array } -function mergeValues(targetPath: string, newValues): void { - let values - if (fs.existsSync(targetPath)) { - if (targetPath.includes('/secrets.')) { - values = yaml.load(fs.readFileSync(`${targetPath}.dec`).toString()) - } else { - values = yaml.load(fs.readFileSync(targetPath).toString()) - } - - if (!values) { - values = {} - } - - merge(values, newValues) - - if (targetPath.includes('/secrets.')) { - fs.writeFileSync(`${targetPath}.dec`, yaml.safeDump(values)) - } else { - fs.writeFileSync(targetPath, yaml.safeDump(values)) - } - } else { - // if the targetPath doesn't exist, just create it and write the valueObject on it. Doesn't matter if it is secret or not. and always write in its yaml file +function mergeValues(targetPath: string, inValues: object): void { + const newValues = cleanValues(inValues) + console.debug(`targetPath: ${targetPath}, values: ${JSON.stringify(newValues)}`) + if (!fs.existsSync(targetPath)) { + // If the targetPath doesn't exist, just create it and write the valueObject in it. + // It doesn't matter if it is secret or not. and always write in its yaml file fs.writeFileSync(targetPath, yaml.safeDump(newValues)) + return } + let useSuffix = suffix + if (!targetPath.includes('/secrets.')) useSuffix = '' + const values = cleanValues(yaml.load(fs.readFileSync(`${targetPath}${useSuffix}`).toString())) + merge(values, newValues) + fs.writeFileSync(`${targetPath}${useSuffix}`, yaml.safeDump(values)) } export default async function main(): Promise { diff --git a/src/utils.test.ts b/src/utils.test.ts index 60e5901b..bd2712b8 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -4,7 +4,7 @@ import sinon from 'sinon' import { expect } from 'chai' import { cloneDeep } from 'lodash' import http from 'http' -import { createPullSecret, deletePullSecret, getApiClient, objectToArray } from './utils' +import { cleanValues, createPullSecret, deletePullSecret, getApiClient, objectToArray } from './utils' describe('Utils', () => { it('should convert an object to array', (done) => { @@ -116,4 +116,11 @@ describe('Secret creation', () => { expect(patchSpy).to.have.been.calledWith('default', namespace, saNewEmpty) expect(deleteSpy).to.have.been.calledWith(name, namespace) }) + + it('should clean an object successfully', async () => { + const inp = { teamConfig: { teams: null } } + const out = { teamConfig: {} } + const res = cleanValues(inp) + expect(res).to.deep.equal(out) + }) }) diff --git a/src/utils.ts b/src/utils.ts index f808a276..84806f53 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,5 @@ import http from 'http' -import { findIndex, mapValues } from 'lodash' +import { findIndex, isNil, mapValues, omitBy } from 'lodash' import { CoreV1Api, KubeConfig, V1Secret, V1ObjectMeta, V1ServiceAccount } from '@kubernetes/client-node' let apiClient: CoreV1Api @@ -185,3 +185,11 @@ export async function deletePullSecret(teamId: string, name: string): Promise { + if (typeof obj[k] === 'object') obj[k] = cleanValues(obj[k]) + }) + return obj +} From 75ec9a34b11a42a4791989cc8c17a9e546e4613a Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 8 Jul 2021 23:09:26 +0200 Subject: [PATCH 20/20] fix: console logging [ci skip] --- .eslintrc.yml | 2 +- src/tasks/otomi/otomi-chart.ts | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.eslintrc.yml b/.eslintrc.yml index efb99f91..a57321e9 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -43,6 +43,6 @@ rules: no-unused-vars: error prefer-destructuring: error no-use-before-define: error - no-console: off + no-console: warn object-shorthand: error no-debugger: error diff --git a/src/tasks/otomi/otomi-chart.ts b/src/tasks/otomi/otomi-chart.ts index 3cd1497d..d010373d 100644 --- a/src/tasks/otomi/otomi-chart.ts +++ b/src/tasks/otomi/otomi-chart.ts @@ -33,7 +33,7 @@ export function extractSecrets(schema: any, parentAddress?: string): Array { const cleanSchema = omit(derefSchema, ['definitions', 'properties.teamConfig']) // FIXME: lets fix the team part later const secretsJsonPath = extractSecrets(cleanSchema) const secrets = pick(values, secretsJsonPath) - console.log(secretsJsonPath) - console.log(secrets) // mergeValues(`${env.OTOMI_ENV_DIR}/env/secrets.teams.yaml`, { teamConfig: secrets.teamConfig }) // FIXME: lets fix the team part later const secretSettings = omit(secrets, ['cluster', 'policies', 'teamConfig', 'charts'])