From a5ec62be7356344c81b810962e2f7c9da99bae90 Mon Sep 17 00:00:00 2001 From: vponline Date: Wed, 21 Dec 2022 16:35:24 +0800 Subject: [PATCH 01/11] Add check for missing airnode deployment files --- .../src/infrastructure/index.ts | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/airnode-deployer/src/infrastructure/index.ts b/packages/airnode-deployer/src/infrastructure/index.ts index d115ebeed2..709a089b8b 100644 --- a/packages/airnode-deployer/src/infrastructure/index.ts +++ b/packages/airnode-deployer/src/infrastructure/index.ts @@ -460,18 +460,43 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy } const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; + const latestDepolymentFileNames = Object.keys((stageDirectory.children[latestDeployment] as Directory).children); + + const requiredFileNames = ['config.json', 'secrets.env', 'default.tfstate']; + const missingRequiredFiles = requiredFileNames.filter( + (requiredFileName) => !latestDepolymentFileNames.includes(requiredFileName) + ); + if (missingRequiredFiles.length) { + logger.warn( + `Airnode '${airnodeAddress}' with stage '${stage}' is missing files: ${missingRequiredFiles.join( + ', ' + )}. Deployer commands may fail and manual removal may be necessary.` + ); + } + const bucketLatestDeploymentPath = `${airnodeAddress}/${stage}/${latestDeployment}`; const bucketConfigPath = `${bucketLatestDeploymentPath}/config.json`; logger.debug(`Fetching configuration file '${bucketConfigPath}'`); - const config = JSON.parse( - await cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketConfigPath) + const goGetConfigFileFromBucket = await go(() => + cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketConfigPath) ); + if (!goGetConfigFileFromBucket.success) { + logger.warn(`Failed to fetch configuration file. Error: ${goGetConfigFileFromBucket.error.message} Skipping.`); + continue; + } + const config = JSON.parse(goGetConfigFileFromBucket.data); + logger.debug(`Fetching secrets file '${bucketConfigPath}'`); const bucketSecretsPath = `${bucketLatestDeploymentPath}/secrets.env`; - const secrets = dotenv.parse( - await cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketSecretsPath) + const goGetSecretsFileFromBucket = await go(() => + cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketSecretsPath) ); + if (!goGetSecretsFileFromBucket.success) { + logger.warn(`Failed to fetch secrets file. Error: ${goGetSecretsFileFromBucket.error.message} Skipping.`); + continue; + } + const secrets = dotenv.parse(goGetSecretsFileFromBucket.data); const interpolatedConfig = unsafeParseConfigWithSecrets(config, secrets); const cloudProvider = interpolatedConfig.nodeSettings.cloudProvider as CloudProvider; From cdd8d03856dbe02ffdddef114bbe9fc8dbc7bb90 Mon Sep 17 00:00:00 2001 From: vponline Date: Wed, 21 Dec 2022 16:36:13 +0800 Subject: [PATCH 02/11] Enable ora warn and info logging while spinner is spinning --- packages/airnode-deployer/src/utils/logger.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/airnode-deployer/src/utils/logger.ts b/packages/airnode-deployer/src/utils/logger.ts index 656e971e87..7548a46edc 100644 --- a/packages/airnode-deployer/src/utils/logger.ts +++ b/packages/airnode-deployer/src/utils/logger.ts @@ -36,15 +36,25 @@ export function fail(text: string) { } export function warn(text: string) { + const currentOra = getSpinner(); + if (currentOra.isSpinning) { + currentOra.clear(); + currentOra.frame(); + } oraInstance().warn(text); } export function info(text: string) { + const currentOra = getSpinner(); + if (currentOra.isSpinning) { + currentOra.clear(); + currentOra.frame(); + } oraInstance().info(text); } export function debug(text: string) { - if (debugModeFlag) oraInstance().info(text); + if (debugModeFlag) info(text); } export function debugSpinner(text: string) { From 9d7623cadff8cb62a32ae3474a87d750ebb9d33e Mon Sep 17 00:00:00 2001 From: vponline Date: Wed, 21 Dec 2022 16:38:25 +0800 Subject: [PATCH 03/11] Add changeset --- .changeset/mighty-dryers-cheat.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/mighty-dryers-cheat.md diff --git a/.changeset/mighty-dryers-cheat.md b/.changeset/mighty-dryers-cheat.md new file mode 100644 index 0000000000..e3484b1666 --- /dev/null +++ b/.changeset/mighty-dryers-cheat.md @@ -0,0 +1,5 @@ +--- +'@api3/airnode-deployer': patch +--- + +Add check for missing secrets.env, config.json, default.tfstate files in bucket From 790080ab8ffac4f272b26d4131e40b568948da37 Mon Sep 17 00:00:00 2001 From: vponline Date: Wed, 21 Dec 2022 17:34:06 +0800 Subject: [PATCH 04/11] Fix missing latestDeployment children object --- packages/airnode-deployer/src/infrastructure/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/airnode-deployer/src/infrastructure/index.ts b/packages/airnode-deployer/src/infrastructure/index.ts index 709a089b8b..ce683acf1a 100644 --- a/packages/airnode-deployer/src/infrastructure/index.ts +++ b/packages/airnode-deployer/src/infrastructure/index.ts @@ -460,7 +460,10 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy } const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; - const latestDepolymentFileNames = Object.keys((stageDirectory.children[latestDeployment] as Directory).children); + // S + const latestDepolymentFileNames = Object.keys( + (stageDirectory.children[latestDeployment] as Directory)?.children || {} + ); const requiredFileNames = ['config.json', 'secrets.env', 'default.tfstate']; const missingRequiredFiles = requiredFileNames.filter( From c05171c8312980d00d2003379b0d0993ddc79d6f Mon Sep 17 00:00:00 2001 From: vponline Date: Thu, 22 Dec 2022 20:32:27 +0800 Subject: [PATCH 05/11] Skip fetching deployment files if config.json or secrets.env is missing --- packages/airnode-deployer/src/infrastructure/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/airnode-deployer/src/infrastructure/index.ts b/packages/airnode-deployer/src/infrastructure/index.ts index ce683acf1a..a46d857d0d 100644 --- a/packages/airnode-deployer/src/infrastructure/index.ts +++ b/packages/airnode-deployer/src/infrastructure/index.ts @@ -460,7 +460,6 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy } const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; - // S const latestDepolymentFileNames = Object.keys( (stageDirectory.children[latestDeployment] as Directory)?.children || {} ); @@ -475,6 +474,10 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy ', ' )}. Deployer commands may fail and manual removal may be necessary.` ); + + if (missingRequiredFiles.includes('config.json') || missingRequiredFiles.includes('secrets.env')) { + continue; + } } const bucketLatestDeploymentPath = `${airnodeAddress}/${stage}/${latestDeployment}`; From 64d38cc962492994dafbc2b7ed4156269fc0efc0 Mon Sep 17 00:00:00 2001 From: vponline Date: Mon, 9 Jan 2023 17:14:42 +0800 Subject: [PATCH 06/11] Refactor checking for bucket missing files, add check to deployAirnode --- .../src/infrastructure/index.ts | 32 +++++------ .../src/utils/infrastructure.ts | 57 ++++++++++++++++++- 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/packages/airnode-deployer/src/infrastructure/index.ts b/packages/airnode-deployer/src/infrastructure/index.ts index a46d857d0d..99051ce508 100644 --- a/packages/airnode-deployer/src/infrastructure/index.ts +++ b/packages/airnode-deployer/src/infrastructure/index.ts @@ -35,6 +35,7 @@ import { FileSystemType, deploymentComparator, Bucket, + checkBucketMissingFiles, } from '../utils/infrastructure'; import { version as nodeVersion } from '../../package.json'; import { deriveAirnodeAddress } from '../utils'; @@ -332,6 +333,16 @@ export const deployAirnode = async (config: Config, configPath: string, secretsP const stageDirectory = getStageDirectory(directoryStructure, airnodeAddress, stage); if (stageDirectory) { logger.debug(`Deployment '${bucketStagePath}' already exists`); + + const bucketMissingFiles = checkBucketMissingFiles(directoryStructure, bucket, type); + if (bucketMissingFiles[airnodeAddress] && bucketMissingFiles[airnodeAddress][stage]) { + throw new Error( + `Can't update an Airnode with missing files: ${bucketMissingFiles[airnodeAddress][stage].join( + ', ' + )}. Deployer commands may fail and manual removal may be necessary.` + ); + } + const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; const bucketConfigPath = `${bucketStagePath}/${latestDeployment}/config.json`; logger.debug(`Fetching configuration file '${bucketConfigPath}'`); @@ -439,6 +450,8 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy } const directoryStructure = await cloudProviderLib[cloudProviderType].getBucketDirectoryStructure(bucket.name); + const bucketMissingFiles = checkBucketMissingFiles(directoryStructure, bucket, cloudProviderType); + for (const [airnodeAddress, addressDirectory] of Object.entries(directoryStructure)) { if (addressDirectory.type !== FileSystemType.Directory) { logger.warn( @@ -460,24 +473,9 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy } const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; - const latestDepolymentFileNames = Object.keys( - (stageDirectory.children[latestDeployment] as Directory)?.children || {} - ); - const requiredFileNames = ['config.json', 'secrets.env', 'default.tfstate']; - const missingRequiredFiles = requiredFileNames.filter( - (requiredFileName) => !latestDepolymentFileNames.includes(requiredFileName) - ); - if (missingRequiredFiles.length) { - logger.warn( - `Airnode '${airnodeAddress}' with stage '${stage}' is missing files: ${missingRequiredFiles.join( - ', ' - )}. Deployer commands may fail and manual removal may be necessary.` - ); - - if (missingRequiredFiles.includes('config.json') || missingRequiredFiles.includes('secrets.env')) { - continue; - } + if (bucketMissingFiles[airnodeAddress] && bucketMissingFiles[airnodeAddress][stage].length) { + continue; } const bucketLatestDeploymentPath = `${airnodeAddress}/${stage}/${latestDeployment}`; diff --git a/packages/airnode-deployer/src/utils/infrastructure.ts b/packages/airnode-deployer/src/utils/infrastructure.ts index b81cbf5f7b..0c6a0d1d69 100644 --- a/packages/airnode-deployer/src/utils/infrastructure.ts +++ b/packages/airnode-deployer/src/utils/infrastructure.ts @@ -1,11 +1,16 @@ import { randomBytes } from 'crypto'; import isArray from 'lodash/isArray'; +import isEmpty from 'lodash/isEmpty'; +import difference from 'lodash/difference'; import { compareVersions } from 'compare-versions'; +import { CloudProvider } from '@api3/airnode-node'; import * as logger from './logger'; -import { Deployment } from '../infrastructure'; +import { Deployment, TF_STATE_FILENAME } from '../infrastructure'; type CommandArg = string | [string, string] | [string, string, string]; +const requiredFileNames = ['config.json', 'secrets.env', TF_STATE_FILENAME]; + export function formatTerraformArguments(args: CommandArg[]) { return args .map((arg) => { @@ -163,3 +168,53 @@ export const deploymentComparator = (a: Deployment, b: Deployment) => { return compareVersions(a.airnodeVersion, b.airnodeVersion); }; + +export const checkBucketMissingFiles = ( + directoryStructure: DirectoryStructure, + bucket: { + name: string; + region: string; + }, + cloudProviderType: CloudProvider['type'] +): Record> => + Object.entries(directoryStructure).reduce((acc, [airnodeAddress, addressDirectory]) => { + if (addressDirectory.type !== FileSystemType.Directory) { + logger.warn( + `Invalid item in bucket '${bucket.name}' (${cloudProviderType.toUpperCase()}) with key '${ + addressDirectory.bucketKey + }'. Skipping.` + ); + return acc; + } + + const checkedAddressDirectory = Object.entries(addressDirectory.children).reduce((acc, [stage, stageDirectory]) => { + if (stageDirectory.type !== FileSystemType.Directory) { + logger.warn( + `Invalid item in bucket '${bucket.name}' (${cloudProviderType.toUpperCase()}) with key '${ + stageDirectory.bucketKey + }'. Skipping.` + ); + return acc; + } + + const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; + const latestDepolymentFileNames = Object.keys( + (stageDirectory.children[latestDeployment] as Directory)?.children || {} + ); + + const missingRequiredFiles = difference(requiredFileNames, latestDepolymentFileNames); + if (isEmpty(missingRequiredFiles)) { + return { ...acc, [airnodeAddress]: { [stage]: [] } }; + } + + logger.warn( + `Airnode '${airnodeAddress}' with stage '${stage}' is missing files: ${missingRequiredFiles.join( + ', ' + )}. Deployer commands may fail and manual removal may be necessary.` + ); + + return { ...acc, [stage]: missingRequiredFiles }; + }, {}); + + return { ...acc, [airnodeAddress]: checkedAddressDirectory }; + }, {}); From 3d0f96715ad8839519f146c48476bab4bd442d99 Mon Sep 17 00:00:00 2001 From: vponline Date: Mon, 9 Jan 2023 18:59:42 +0800 Subject: [PATCH 07/11] Fix missing file check --- .../airnode-deployer/src/infrastructure/index.ts | 12 ++++++++++-- .../airnode-deployer/src/utils/infrastructure.ts | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/airnode-deployer/src/infrastructure/index.ts b/packages/airnode-deployer/src/infrastructure/index.ts index 99051ce508..716130b790 100644 --- a/packages/airnode-deployer/src/infrastructure/index.ts +++ b/packages/airnode-deployer/src/infrastructure/index.ts @@ -335,7 +335,11 @@ export const deployAirnode = async (config: Config, configPath: string, secretsP logger.debug(`Deployment '${bucketStagePath}' already exists`); const bucketMissingFiles = checkBucketMissingFiles(directoryStructure, bucket, type); - if (bucketMissingFiles[airnodeAddress] && bucketMissingFiles[airnodeAddress][stage]) { + if ( + bucketMissingFiles[airnodeAddress] && + bucketMissingFiles[airnodeAddress][stage] && + bucketMissingFiles[airnodeAddress][stage].length !== 0 + ) { throw new Error( `Can't update an Airnode with missing files: ${bucketMissingFiles[airnodeAddress][stage].join( ', ' @@ -474,7 +478,11 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; - if (bucketMissingFiles[airnodeAddress] && bucketMissingFiles[airnodeAddress][stage].length) { + if ( + bucketMissingFiles[airnodeAddress] && + bucketMissingFiles[airnodeAddress][stage] && + bucketMissingFiles[airnodeAddress][stage].length !== 0 + ) { continue; } diff --git a/packages/airnode-deployer/src/utils/infrastructure.ts b/packages/airnode-deployer/src/utils/infrastructure.ts index 0c6a0d1d69..5723ff7669 100644 --- a/packages/airnode-deployer/src/utils/infrastructure.ts +++ b/packages/airnode-deployer/src/utils/infrastructure.ts @@ -216,5 +216,5 @@ export const checkBucketMissingFiles = ( return { ...acc, [stage]: missingRequiredFiles }; }, {}); - return { ...acc, [airnodeAddress]: checkedAddressDirectory }; + return { ...acc, ...checkedAddressDirectory }; }, {}); From 45c9c7ebd0292956583a5028f7712f93312fe89a Mon Sep 17 00:00:00 2001 From: vponline Date: Mon, 9 Jan 2023 20:01:17 +0800 Subject: [PATCH 08/11] Fix missing files object keys --- packages/airnode-deployer/src/utils/infrastructure.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/airnode-deployer/src/utils/infrastructure.ts b/packages/airnode-deployer/src/utils/infrastructure.ts index 5723ff7669..a97d8403c1 100644 --- a/packages/airnode-deployer/src/utils/infrastructure.ts +++ b/packages/airnode-deployer/src/utils/infrastructure.ts @@ -213,7 +213,7 @@ export const checkBucketMissingFiles = ( )}. Deployer commands may fail and manual removal may be necessary.` ); - return { ...acc, [stage]: missingRequiredFiles }; + return { ...acc, [airnodeAddress]: { [stage]: missingRequiredFiles } }; }, {}); return { ...acc, ...checkedAddressDirectory }; From 569434596d56dbb6ea7c0967e4b1b63b5433b9cf Mon Sep 17 00:00:00 2001 From: vponline Date: Mon, 9 Jan 2023 20:44:19 +0800 Subject: [PATCH 09/11] Remove use of TF_STATE_FILENAME to fix tests --- packages/airnode-deployer/src/utils/infrastructure.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/airnode-deployer/src/utils/infrastructure.ts b/packages/airnode-deployer/src/utils/infrastructure.ts index a97d8403c1..faabe4a0ee 100644 --- a/packages/airnode-deployer/src/utils/infrastructure.ts +++ b/packages/airnode-deployer/src/utils/infrastructure.ts @@ -5,11 +5,11 @@ import difference from 'lodash/difference'; import { compareVersions } from 'compare-versions'; import { CloudProvider } from '@api3/airnode-node'; import * as logger from './logger'; -import { Deployment, TF_STATE_FILENAME } from '../infrastructure'; +import { Deployment } from '../infrastructure'; type CommandArg = string | [string, string] | [string, string, string]; -const requiredFileNames = ['config.json', 'secrets.env', TF_STATE_FILENAME]; +const requiredFileNames = ['config.json', 'secrets.env', 'default.tfstate']; export function formatTerraformArguments(args: CommandArg[]) { return args From 410b315076c7e4ee459bbc04b1add0b68f1c9b2b Mon Sep 17 00:00:00 2001 From: vponline Date: Tue, 10 Jan 2023 16:50:19 +0800 Subject: [PATCH 10/11] Remove duplicate logging of invalid bucket directory --- .../src/infrastructure/index.ts | 4 ++-- .../src/utils/infrastructure.ts | 18 +----------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/packages/airnode-deployer/src/infrastructure/index.ts b/packages/airnode-deployer/src/infrastructure/index.ts index 716130b790..10078b9b47 100644 --- a/packages/airnode-deployer/src/infrastructure/index.ts +++ b/packages/airnode-deployer/src/infrastructure/index.ts @@ -334,7 +334,7 @@ export const deployAirnode = async (config: Config, configPath: string, secretsP if (stageDirectory) { logger.debug(`Deployment '${bucketStagePath}' already exists`); - const bucketMissingFiles = checkBucketMissingFiles(directoryStructure, bucket, type); + const bucketMissingFiles = checkBucketMissingFiles(directoryStructure); if ( bucketMissingFiles[airnodeAddress] && bucketMissingFiles[airnodeAddress][stage] && @@ -454,7 +454,7 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy } const directoryStructure = await cloudProviderLib[cloudProviderType].getBucketDirectoryStructure(bucket.name); - const bucketMissingFiles = checkBucketMissingFiles(directoryStructure, bucket, cloudProviderType); + const bucketMissingFiles = checkBucketMissingFiles(directoryStructure); for (const [airnodeAddress, addressDirectory] of Object.entries(directoryStructure)) { if (addressDirectory.type !== FileSystemType.Directory) { diff --git a/packages/airnode-deployer/src/utils/infrastructure.ts b/packages/airnode-deployer/src/utils/infrastructure.ts index faabe4a0ee..ec9c94f7a1 100644 --- a/packages/airnode-deployer/src/utils/infrastructure.ts +++ b/packages/airnode-deployer/src/utils/infrastructure.ts @@ -3,7 +3,6 @@ import isArray from 'lodash/isArray'; import isEmpty from 'lodash/isEmpty'; import difference from 'lodash/difference'; import { compareVersions } from 'compare-versions'; -import { CloudProvider } from '@api3/airnode-node'; import * as logger from './logger'; import { Deployment } from '../infrastructure'; @@ -170,30 +169,15 @@ export const deploymentComparator = (a: Deployment, b: Deployment) => { }; export const checkBucketMissingFiles = ( - directoryStructure: DirectoryStructure, - bucket: { - name: string; - region: string; - }, - cloudProviderType: CloudProvider['type'] + directoryStructure: DirectoryStructure ): Record> => Object.entries(directoryStructure).reduce((acc, [airnodeAddress, addressDirectory]) => { if (addressDirectory.type !== FileSystemType.Directory) { - logger.warn( - `Invalid item in bucket '${bucket.name}' (${cloudProviderType.toUpperCase()}) with key '${ - addressDirectory.bucketKey - }'. Skipping.` - ); return acc; } const checkedAddressDirectory = Object.entries(addressDirectory.children).reduce((acc, [stage, stageDirectory]) => { if (stageDirectory.type !== FileSystemType.Directory) { - logger.warn( - `Invalid item in bucket '${bucket.name}' (${cloudProviderType.toUpperCase()}) with key '${ - stageDirectory.bucketKey - }'. Skipping.` - ); return acc; } From e5de4dea67ec19e93bfe95c85d98d86e8b378de6 Mon Sep 17 00:00:00 2001 From: vponline Date: Tue, 10 Jan 2023 20:38:34 +0800 Subject: [PATCH 11/11] Wrap file fetching and JSON.parse with go, refactor names --- .../src/infrastructure/index.ts | 29 +++++++++++++------ .../src/utils/infrastructure.ts | 6 ++-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/packages/airnode-deployer/src/infrastructure/index.ts b/packages/airnode-deployer/src/infrastructure/index.ts index fd43176a1e..9c599c91ee 100644 --- a/packages/airnode-deployer/src/infrastructure/index.ts +++ b/packages/airnode-deployer/src/infrastructure/index.ts @@ -36,7 +36,7 @@ import { FileSystemType, deploymentComparator, Bucket, - checkBucketMissingFiles, + getMissingBucketFiles, } from '../utils/infrastructure'; import { version as nodeVersion } from '../../package.json'; import { deriveAirnodeAddress } from '../utils'; @@ -335,7 +335,7 @@ export const deployAirnode = async (config: Config, configPath: string, secretsP if (stageDirectory) { logger.debug(`Deployment '${bucketStagePath}' already exists`); - const bucketMissingFiles = checkBucketMissingFiles(directoryStructure); + const bucketMissingFiles = getMissingBucketFiles(directoryStructure); if ( bucketMissingFiles[airnodeAddress] && bucketMissingFiles[airnodeAddress][stage] && @@ -351,11 +351,18 @@ export const deployAirnode = async (config: Config, configPath: string, secretsP const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; const bucketConfigPath = `${bucketStagePath}/${latestDeployment}/config.json`; logger.debug(`Fetching configuration file '${bucketConfigPath}'`); - const remoteConfig = JSON.parse( - await cloudProviderLib[type].getFileFromBucket(bucket.name, bucketConfigPath) - ) as Config; + const goGetRemoteConfigFileFromBucket = await go(() => + cloudProviderLib[type].getFileFromBucket(bucket!.name, bucketConfigPath) + ); + if (!goGetRemoteConfigFileFromBucket.success) { + throw new Error(`Failed to fetch configuration file. Error: ${goGetRemoteConfigFileFromBucket.error.message}`); + } + const goRemoteConfig = goSync(() => JSON.parse(goGetRemoteConfigFileFromBucket.data)); + if (!goRemoteConfig.success) { + throw new Error(`Failed to parse configuration file. Error: ${goRemoteConfig.error.message}`); + } - const remoteNodeSettings = remoteConfig.nodeSettings; + const remoteNodeSettings = goRemoteConfig.data.nodeSettings; const remoteCloudProvider = remoteNodeSettings.cloudProvider as CloudProvider; if (remoteNodeSettings.nodeVersion !== nodeVersion) { throw new Error( @@ -455,7 +462,7 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy } const directoryStructure = await cloudProviderLib[cloudProviderType].getBucketDirectoryStructure(bucket.name); - const bucketMissingFiles = checkBucketMissingFiles(directoryStructure); + const bucketMissingFiles = getMissingBucketFiles(directoryStructure); for (const [airnodeAddress, addressDirectory] of Object.entries(directoryStructure)) { if (addressDirectory.type !== FileSystemType.Directory) { @@ -498,7 +505,11 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy logger.warn(`Failed to fetch configuration file. Error: ${goGetConfigFileFromBucket.error.message} Skipping.`); continue; } - const config = JSON.parse(goGetConfigFileFromBucket.data); + const goConfig = goSync(() => JSON.parse(goGetConfigFileFromBucket.data)); + if (!goConfig.success) { + logger.warn(`Failed to parse configuration file. Error: ${goConfig.error.message} Skipping.`); + continue; + } logger.debug(`Fetching secrets file '${bucketConfigPath}'`); const bucketSecretsPath = `${bucketLatestDeploymentPath}/secrets.env`; @@ -510,7 +521,7 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy continue; } const secrets = dotenv.parse(goGetSecretsFileFromBucket.data); - const interpolatedConfig = unsafeParseConfigWithSecrets(config, secrets); + const interpolatedConfig = unsafeParseConfigWithSecrets(goConfig.data, secrets); const cloudProvider = interpolatedConfig.nodeSettings.cloudProvider as CloudProvider; const airnodeVersion = interpolatedConfig.nodeSettings.nodeVersion; diff --git a/packages/airnode-deployer/src/utils/infrastructure.ts b/packages/airnode-deployer/src/utils/infrastructure.ts index ec9c94f7a1..ff2ac16054 100644 --- a/packages/airnode-deployer/src/utils/infrastructure.ts +++ b/packages/airnode-deployer/src/utils/infrastructure.ts @@ -8,7 +8,7 @@ import { Deployment } from '../infrastructure'; type CommandArg = string | [string, string] | [string, string, string]; -const requiredFileNames = ['config.json', 'secrets.env', 'default.tfstate']; +const DEPLOYMENT_REQUIRED_FILE_NAMES = ['config.json', 'secrets.env', 'default.tfstate']; export function formatTerraformArguments(args: CommandArg[]) { return args @@ -168,7 +168,7 @@ export const deploymentComparator = (a: Deployment, b: Deployment) => { return compareVersions(a.airnodeVersion, b.airnodeVersion); }; -export const checkBucketMissingFiles = ( +export const getMissingBucketFiles = ( directoryStructure: DirectoryStructure ): Record> => Object.entries(directoryStructure).reduce((acc, [airnodeAddress, addressDirectory]) => { @@ -186,7 +186,7 @@ export const checkBucketMissingFiles = ( (stageDirectory.children[latestDeployment] as Directory)?.children || {} ); - const missingRequiredFiles = difference(requiredFileNames, latestDepolymentFileNames); + const missingRequiredFiles = difference(DEPLOYMENT_REQUIRED_FILE_NAMES, latestDepolymentFileNames); if (isEmpty(missingRequiredFiles)) { return { ...acc, [airnodeAddress]: { [stage]: [] } }; }