diff --git a/.changeset/mighty-dryers-cheat.md b/.changeset/mighty-dryers-cheat.md new file mode 100644 index 0000000000..e3484b1666 --- /dev/null +++ b/.changeset/mighty-dryers-cheat.md @@ -0,0 +1,5 @@ +--- +'@api3/airnode-deployer': patch +--- + +Add check for missing secrets.env, config.json, default.tfstate files in bucket diff --git a/packages/airnode-deployer/src/infrastructure/index.ts b/packages/airnode-deployer/src/infrastructure/index.ts index dbd140ed54..9c599c91ee 100644 --- a/packages/airnode-deployer/src/infrastructure/index.ts +++ b/packages/airnode-deployer/src/infrastructure/index.ts @@ -36,6 +36,7 @@ import { FileSystemType, deploymentComparator, Bucket, + getMissingBucketFiles, } from '../utils/infrastructure'; import { version as nodeVersion } from '../../package.json'; import { deriveAirnodeAddress } from '../utils'; @@ -333,14 +334,35 @@ export const deployAirnode = async (config: Config, configPath: string, secretsP const stageDirectory = getStageDirectory(directoryStructure, airnodeAddress, stage); if (stageDirectory) { logger.debug(`Deployment '${bucketStagePath}' already exists`); + + const bucketMissingFiles = getMissingBucketFiles(directoryStructure); + if ( + bucketMissingFiles[airnodeAddress] && + bucketMissingFiles[airnodeAddress][stage] && + bucketMissingFiles[airnodeAddress][stage].length !== 0 + ) { + throw new Error( + `Can't update an Airnode with missing files: ${bucketMissingFiles[airnodeAddress][stage].join( + ', ' + )}. Deployer commands may fail and manual removal may be necessary.` + ); + } + const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; const bucketConfigPath = `${bucketStagePath}/${latestDeployment}/config.json`; logger.debug(`Fetching configuration file '${bucketConfigPath}'`); - const remoteConfig = JSON.parse( - await cloudProviderLib[type].getFileFromBucket(bucket.name, bucketConfigPath) - ) as Config; + const goGetRemoteConfigFileFromBucket = await go(() => + cloudProviderLib[type].getFileFromBucket(bucket!.name, bucketConfigPath) + ); + if (!goGetRemoteConfigFileFromBucket.success) { + throw new Error(`Failed to fetch configuration file. Error: ${goGetRemoteConfigFileFromBucket.error.message}`); + } + const goRemoteConfig = goSync(() => JSON.parse(goGetRemoteConfigFileFromBucket.data)); + if (!goRemoteConfig.success) { + throw new Error(`Failed to parse configuration file. Error: ${goRemoteConfig.error.message}`); + } - const remoteNodeSettings = remoteConfig.nodeSettings; + const remoteNodeSettings = goRemoteConfig.data.nodeSettings; const remoteCloudProvider = remoteNodeSettings.cloudProvider as CloudProvider; if (remoteNodeSettings.nodeVersion !== nodeVersion) { throw new Error( @@ -440,6 +462,8 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy } const directoryStructure = await cloudProviderLib[cloudProviderType].getBucketDirectoryStructure(bucket.name); + const bucketMissingFiles = getMissingBucketFiles(directoryStructure); + for (const [airnodeAddress, addressDirectory] of Object.entries(directoryStructure)) { if (addressDirectory.type !== FileSystemType.Directory) { logger.warn( @@ -461,19 +485,43 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy } const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; + + if ( + bucketMissingFiles[airnodeAddress] && + bucketMissingFiles[airnodeAddress][stage] && + bucketMissingFiles[airnodeAddress][stage].length !== 0 + ) { + continue; + } + const bucketLatestDeploymentPath = `${airnodeAddress}/${stage}/${latestDeployment}`; const bucketConfigPath = `${bucketLatestDeploymentPath}/config.json`; logger.debug(`Fetching configuration file '${bucketConfigPath}'`); - const config = JSON.parse( - await cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketConfigPath) + const goGetConfigFileFromBucket = await go(() => + cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketConfigPath) ); + if (!goGetConfigFileFromBucket.success) { + logger.warn(`Failed to fetch configuration file. Error: ${goGetConfigFileFromBucket.error.message} Skipping.`); + continue; + } + const goConfig = goSync(() => JSON.parse(goGetConfigFileFromBucket.data)); + if (!goConfig.success) { + logger.warn(`Failed to parse configuration file. Error: ${goConfig.error.message} Skipping.`); + continue; + } + + logger.debug(`Fetching secrets file '${bucketConfigPath}'`); const bucketSecretsPath = `${bucketLatestDeploymentPath}/secrets.env`; - logger.debug(`Fetching secrets file '${bucketSecretsPath}'`); - const secrets = dotenv.parse( - await cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketSecretsPath) + const goGetSecretsFileFromBucket = await go(() => + cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketSecretsPath) ); - const interpolatedConfig = unsafeParseConfigWithSecrets(config, secrets); + if (!goGetSecretsFileFromBucket.success) { + logger.warn(`Failed to fetch secrets file. Error: ${goGetSecretsFileFromBucket.error.message} Skipping.`); + continue; + } + const secrets = dotenv.parse(goGetSecretsFileFromBucket.data); + const interpolatedConfig = unsafeParseConfigWithSecrets(goConfig.data, secrets); const cloudProvider = interpolatedConfig.nodeSettings.cloudProvider as CloudProvider; const airnodeVersion = interpolatedConfig.nodeSettings.nodeVersion; diff --git a/packages/airnode-deployer/src/utils/infrastructure.ts b/packages/airnode-deployer/src/utils/infrastructure.ts index b81cbf5f7b..ff2ac16054 100644 --- a/packages/airnode-deployer/src/utils/infrastructure.ts +++ b/packages/airnode-deployer/src/utils/infrastructure.ts @@ -1,11 +1,15 @@ import { randomBytes } from 'crypto'; import isArray from 'lodash/isArray'; +import isEmpty from 'lodash/isEmpty'; +import difference from 'lodash/difference'; import { compareVersions } from 'compare-versions'; import * as logger from './logger'; import { Deployment } from '../infrastructure'; type CommandArg = string | [string, string] | [string, string, string]; +const DEPLOYMENT_REQUIRED_FILE_NAMES = ['config.json', 'secrets.env', 'default.tfstate']; + export function formatTerraformArguments(args: CommandArg[]) { return args .map((arg) => { @@ -163,3 +167,38 @@ export const deploymentComparator = (a: Deployment, b: Deployment) => { return compareVersions(a.airnodeVersion, b.airnodeVersion); }; + +export const getMissingBucketFiles = ( + directoryStructure: DirectoryStructure +): Record> => + Object.entries(directoryStructure).reduce((acc, [airnodeAddress, addressDirectory]) => { + if (addressDirectory.type !== FileSystemType.Directory) { + return acc; + } + + const checkedAddressDirectory = Object.entries(addressDirectory.children).reduce((acc, [stage, stageDirectory]) => { + if (stageDirectory.type !== FileSystemType.Directory) { + return acc; + } + + const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0]; + const latestDepolymentFileNames = Object.keys( + (stageDirectory.children[latestDeployment] as Directory)?.children || {} + ); + + const missingRequiredFiles = difference(DEPLOYMENT_REQUIRED_FILE_NAMES, latestDepolymentFileNames); + if (isEmpty(missingRequiredFiles)) { + return { ...acc, [airnodeAddress]: { [stage]: [] } }; + } + + logger.warn( + `Airnode '${airnodeAddress}' with stage '${stage}' is missing files: ${missingRequiredFiles.join( + ', ' + )}. Deployer commands may fail and manual removal may be necessary.` + ); + + return { ...acc, [airnodeAddress]: { [stage]: missingRequiredFiles } }; + }, {}); + + return { ...acc, ...checkedAddressDirectory }; + }, {}); diff --git a/packages/airnode-deployer/src/utils/logger.ts b/packages/airnode-deployer/src/utils/logger.ts index 656e971e87..7548a46edc 100644 --- a/packages/airnode-deployer/src/utils/logger.ts +++ b/packages/airnode-deployer/src/utils/logger.ts @@ -36,15 +36,25 @@ export function fail(text: string) { } export function warn(text: string) { + const currentOra = getSpinner(); + if (currentOra.isSpinning) { + currentOra.clear(); + currentOra.frame(); + } oraInstance().warn(text); } export function info(text: string) { + const currentOra = getSpinner(); + if (currentOra.isSpinning) { + currentOra.clear(); + currentOra.frame(); + } oraInstance().info(text); } export function debug(text: string) { - if (debugModeFlag) oraInstance().info(text); + if (debugModeFlag) info(text); } export function debugSpinner(text: string) {