K8s cleanup

lib/api-helper/goal/sdmGoal.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2018 Atomist, Inc.
+ * Copyright © 2020 Atomist, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,22 +35,35 @@ export function goalKeyString(gk: SdmGoalKey): string {
 }
 
 /**
- * Retrieve the goal data
- * Note: this purposely only works if the data field is stringified JSON.
+ * Read and parse goal event data.  If the goal event has no data,
+ * return an empty object.  Note: this purposely only works if the
+ * data field is stringified JSON.
+ *
  * @param sdmGoal
+ * @return JSON parsed goal event data property
  */
 export function goalData(sdmGoal: SdmGoalEvent): any {
+    if (!sdmGoal?.data) {
+        return {};
+    }
+    let data: any;
     try {
-        return JSON.parse(sdmGoal.data || "");
+        data = JSON.parse(sdmGoal.data);
     } catch (e) {
-        throw new Error("Goal data is not stringified JSON");
+        e.message = `Failed to parse goal event data for ${sdmGoal.uniqueName} as JSON '${sdmGoal.data}': ${e.message}`;
+        throw e;
     }
+    return data;
 }
 
 /**
- * Merge the provided data into the goal data
+ * Return a shallow merge the provided `data` and the goal event data
+ * property, parsed as JSON.  Properties in `data` take precedence
+ * over those in the parsed goal event data object.
+ *
  * @param data
  * @param sdmGoal
+ * @return shallow merge of data and SDM goal event data property
  */
 export function mergeGoalData(data: any, sdmGoal: SdmGoalEvent): any {
     return {

lib/api-helper/goal/storeGoals.ts

@@ -21,6 +21,7 @@ import {
 import { RemoteRepoRef } from "@atomist/automation-client/lib/operations/common/RepoId";
 import { MutationNoCacheOptions } from "@atomist/automation-client/lib/spi/graph/GraphClient";
 import * as _ from "lodash";
+import * as omitEmpty from "omit-empty";
 import { sprintf } from "sprintf-js";
 import {
     Goal,
@@ -48,9 +49,6 @@ import {
     UpdateSdmGoalSetMutationVariables,
 } from "../../typings/types";
 
-// tslint:disable-next-line:no-var-requires
-const omitEmpty = require("omit-empty");
-
 export function environmentFromGoal(goal: Goal): string {
     return goal.definition.environment.replace(/\/$/, ""); // remove trailing slash at least
 }

lib/api-helper/misc/project/filteredView.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2019 Atomist, Inc.
+ * Copyright © 2020 Atomist, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,8 +31,7 @@ import * as stream from "stream";
  * @param filter function to filter file paths. Return true to eliminate a file
  * @return {Promise<LocalProject>}
  */
-export function filteredView<P extends Project = Project>(p: Project,
-                                                          filter: (path: string) => boolean): P {
+export function filteredView<P extends Project = Project>(p: Project, filter: (path: string) => boolean): P {
     // Use an ES6 proxy to bring back memories of Spring AOP
     const handler = {
         get: (target, prop) => {
@@ -45,10 +44,9 @@ export function filteredView<P extends Project = Project>(p: Project,
                 return origMethod;
             }
             const decoratedMethod = decorator[prop];
-            return function(...args: any[]): any {
+            return function(this: any, ...args: any[]): any {
                 return !!decoratedMethod ?
                     decoratedMethod.apply(decorator, args) :
-                    // tslint:disable-next-line:no-invalid-this
                     origMethod.apply(this, args);
             };
         },
@@ -62,8 +60,7 @@ export function filteredView<P extends Project = Project>(p: Project,
  */
 class FilteredProject implements Partial<Project> {
 
-    constructor(private readonly project: Project,
-                private readonly  filter: (path: string) => boolean) {
+    constructor(private readonly project: Project, private readonly filter: (path: string) => boolean) {
     }
 
     public getFile(path: string): Promise<File | undefined> {
@@ -92,7 +89,7 @@ class FilteredProject implements Partial<Project> {
      */
     public streamFilesRaw(globPatterns: string[], opts: {}): FileStream {
         const filter = this.filter;
-        const onlyIncludedFilters = new stream.Transform({objectMode: true});
+        const onlyIncludedFilters = new stream.Transform({ objectMode: true });
         onlyIncludedFilters._transform = function(f: any, encoding: string, done: stream.TransformCallback): void {
             if (filter(f.path)) {
                 this.push(f);

lib/api-helper/project/GitHubLazyProjectLoader.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2019 Atomist, Inc.
+ * Copyright © 2020 Atomist, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -79,9 +79,7 @@ class GitHubLazyProject extends AbstractProject implements GitProject, LazyProje
 
     private projectPromise: QueryablePromise<GitProject>;
 
-    constructor(id: RemoteRepoRef,
-                private readonly delegate: ProjectLoader,
-                private readonly params: ProjectLoadingParameters) {
+    constructor(id: RemoteRepoRef, private readonly delegate: ProjectLoader, private readonly params: ProjectLoadingParameters) {
         super(id);
     }
 
@@ -198,8 +196,7 @@ class GitHubLazyProject extends AbstractProject implements GitProject, LazyProje
 
     public streamFilesRaw(globPatterns: string[], opts: {}): FileStream {
         const resultStream = new stream.Transform({ objectMode: true });
-        resultStream._transform = function(chunk: any, encoding: string, done: stream.TransformCallback): void {
-            // tslint:disable-next-line:no-invalid-this
+        resultStream._transform = function(this: stream.Transform, chunk: any, encoding: string, done: stream.TransformCallback): void {
             this.push(chunk);
             done();
         };
@@ -227,9 +224,7 @@ class GitHubLazyProject extends AbstractProject implements GitProject, LazyProje
         return this.projectPromise.then(mp => mp.configureFromRemote()) as any;
     }
 
-    public createAndSetRemote(gid: RemoteRepoRef,
-                              description: string,
-                              visibility: "private" | "public"): Promise<this> {
+    public createAndSetRemote(gid: RemoteRepoRef, description: string, visibility: "private" | "public"): Promise<this> {
         this.materializeIfNecessary("createAndSetRemote");
         return this.projectPromise.then(mp => mp.createAndSetRemote(gid, description, visibility)) as any;
     }

lib/api-helper/project/configuration/projectConfiguration.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2019 Atomist, Inc.
+ * Copyright © 2020 Atomist, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 import { configurationValue } from "@atomist/automation-client/lib/configuration";
 import { Project } from "@atomist/automation-client/lib/project/Project";
 import { logger } from "@atomist/automation-client/lib/util/logger";
-import yaml = require("js-yaml");
+import * as yaml from "js-yaml";
 import * as _ from "lodash";
 
 /**

lib/api/goal/GoalWithFulfillment.ts

@@ -17,6 +17,7 @@
 import { configurationValue } from "@atomist/automation-client/lib/configuration";
 import { RetryOptions } from "@atomist/automation-client/lib/util/retry";
 import * as _ from "lodash";
+import { goalData } from "../../api-helper/goal/sdmGoal";
 import { LogSuppressor } from "../../api-helper/log/logInterpreters";
 import { AbstractSoftwareDeliveryMachine } from "../../api-helper/machine/AbstractSoftwareDeliveryMachine";
 import { InterpretLog } from "../../spi/log/InterpretedLog";
@@ -223,7 +224,7 @@ export abstract class FulfillableGoal extends GoalWithPrecondition implements Re
             callback: async (goalEvent, repoContext) => {
                 const service = await registration.service(goalEvent, repoContext);
                 if (!!service) {
-                    const data = JSON.parse(goalEvent.data || "{}");
+                    const data = goalData(goalEvent);
                     const servicesData = {};
                     _.set<any>(servicesData, `${ServiceRegistrationGoalDataKey}.${registration.name}`, service);
                     goalEvent.data = JSON.stringify(_.merge(data, servicesData));

lib/api/mapping/goalTest.ts

@@ -17,6 +17,7 @@
 import { AutomationContextAware } from "@atomist/automation-client/lib/HandlerContext";
 import { isEventIncoming } from "@atomist/automation-client/lib/internal/transport/RequestProcessor";
 import * as _ from "lodash";
+import { goalData } from "../../api-helper/goal/sdmGoal";
 import { SdmGoalState } from "../../typings/types";
 import { StatefulPushListenerInvocation } from "../dsl/goalContribution";
 import { SdmGoalEvent } from "../goal/SdmGoalEvent";
@@ -51,7 +52,7 @@ export function isGoal(options: {
                 return false;
             }
             if (!!options.output) {
-                const data = JSON.parse(g.data || "{}");
+                const data = goalData(g);
                 const outputs: Array<{ classifier: string }> = data["@atomist/sdm/output"];
                 if (!outputs) {
                     return false;

lib/core/goal/cache/goalCaching.ts

@@ -20,6 +20,7 @@ import { GitProject } from "@atomist/automation-client/lib/project/git/GitProjec
 import { Project } from "@atomist/automation-client/lib/project/Project";
 import { gatherFromFiles } from "@atomist/automation-client/lib/project/util/projectUtils";
 import * as _ from "lodash";
+import { goalData } from "../../../api-helper/goal/sdmGoal";
 import { ExecuteGoalResult } from "../../../api/goal/ExecuteGoalResult";
 import {
     GoalInvocation,
@@ -147,8 +148,7 @@ export function cachePut(options: GoalCacheOptions,
 
     return {
         name: listenerName,
-        listener: async (p: GitProject,
-                         gi: GoalInvocation): Promise<void | ExecuteGoalResult> => {
+        listener: async (p: GitProject, gi: GoalInvocation): Promise<void | ExecuteGoalResult> => {
             const { goalEvent } = gi;
             if (!!isCacheEnabled(gi) && !process.env.ATOMIST_ISOLATED_GOAL_INIT) {
                 const cloneEntries = _.cloneDeep(entries);
@@ -171,15 +171,15 @@ export function cachePut(options: GoalCacheOptions,
                 }
 
                 // Set outputs on the goal data
-                const data = JSON.parse(goalEvent.data || "{}");
+                const data = goalData(goalEvent);
                 const newData = {
                     [CacheOutputGoalDataKey]: [
                         ...(data[CacheOutputGoalDataKey] || []),
                         ...cloneEntries,
                     ],
                 };
                 goalEvent.data = JSON.stringify({
-                    ...(JSON.parse(goalEvent.data || "{}")),
+                    ...(goalData(goalEvent)),
                     ...newData,
                 });
             }
@@ -281,7 +281,7 @@ export function cacheRestore(options: GoalCacheRestoreOptions,
 
             // Set inputs on the goal data
             const { goalEvent } = gi;
-            const data = JSON.parse(goalEvent.data || "{}");
+            const data = goalData(goalEvent);
             const newData = {
                 [CacheInputGoalDataKey]: [
                     ...(data[CacheInputGoalDataKey] || []),
@@ -291,7 +291,7 @@ export function cacheRestore(options: GoalCacheRestoreOptions,
                 ],
             };
             goalEvent.data = JSON.stringify({
-                ...(JSON.parse(goalEvent.data || "{}")),
+                ...(goalData(goalEvent)),
                 ...newData,
             });
         },

lib/core/goal/container/container.ts

@@ -274,8 +274,6 @@ export class Container extends FulfillableGoalWithRegistrations<ContainerRegistr
             if (!goalSchedulers.some(gs => gs instanceof kgs.KubernetesGoalScheduler)) {
                 if (!process.env.ATOMIST_ISOLATED_GOAL && kgs.isConfiguredInEnv("kubernetes", "kubernetes-all")) {
                     sdm.configuration.sdm.goalScheduler = [...goalSchedulers, new kgs.KubernetesGoalScheduler()];
-                    const kjdgcl = require("../../pack/k8s/scheduler/KubernetesJobDeletingGoalCompletionListener");
-                    sdm.addGoalCompletionListener(new kjdgcl.KubernetesJobDeletingGoalCompletionListenerFactory(sdm).create());
                 }
             }
         } else if (runningAsGoogleCloudFunction()) {
@@ -295,10 +293,10 @@ export class Container extends FulfillableGoalWithRegistrations<ContainerRegistr
         registration.name = (registration.name || `container-${this.definition.displayName}`).replace(/\.+/g, "-");
         if (!this.details.scheduler) {
             if (runningInK8s()) {
-                const k8sContainerScheduler = require("./k8s").k8sContainerScheduler;
+                const k8sContainerScheduler = require("../../pack/k8s/container").k8sContainerScheduler;
                 this.details.scheduler = k8sContainerScheduler;
             } else if (runningAsGoogleCloudFunction()) {
-                const k8sSkillContainerScheduler = require("./k8s").k8sSkillContainerScheduler;
+                const k8sSkillContainerScheduler = require("../../pack/k8s/container").k8sSkillContainerScheduler;
                 this.details.scheduler = k8sSkillContainerScheduler;
             } else {
                 this.details.scheduler = dockerContainerScheduler;

lib/core/goal/container/util.ts

@@ -15,15 +15,13 @@
  */
 
 import { MutationNoCacheOptions } from "@atomist/automation-client/lib/spi/graph/GraphClient";
-import { LeveledLogMethod } from "@atomist/automation-client/lib/util/logger";
 import * as fs from "fs-extra";
 import * as _ from "lodash";
 import * as path from "path";
 import { SdmContext } from "../../../api/context/SdmContext";
 import { ExecuteGoalResult } from "../../../api/goal/ExecuteGoalResult";
 import { GoalInvocation } from "../../../api/goal/GoalInvocation";
 import { SdmGoalEvent } from "../../../api/goal/SdmGoalEvent";
-import { ProgressLog } from "../../../spi/log/ProgressLog";
 import {
     OnBuildCompleteForDryRun,
     PushFields,
@@ -155,20 +153,7 @@ export async function prepareInputAndOutput(input: string, output: string, gi: G
     }
 }
 
-/**
- * Write to client and progress logs.  Add newline to progress log.
- *
- * @param msg Message to write, should not have newline at end
- * @param l Logger method, e.g., `logger.warn`
- * @param p Progress log
- */
-export function loglog(msg: string, l: LeveledLogMethod, p: ProgressLog): void {
-    l(msg);
-    p.write(msg + "\n");
-}
-
-export async function processResult(result: any,
-                                    gi: GoalInvocation): Promise<ExecuteGoalResult | undefined> {
+export async function processResult(result: any, gi: GoalInvocation): Promise<ExecuteGoalResult | undefined> {
     const { goalEvent, context } = gi;
     if (!!result) {
         if (result.SdmGoal) {

lib/core/goal/skillOutput.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { goalData } from "../../api-helper/goal/sdmGoal";
 import { GoalExecutionListener } from "../../api/listener/GoalStatusListener";
 import {
     CustomSkillOutputInput,
@@ -42,7 +43,7 @@ export const SkillOutputGoalExecutionListener: GoalExecutionListener = async gi
         return;
     }
 
-    const data = JSON.parse(goalEvent.data || "{}");
+    const data = goalData(goalEvent);
     const entries: Array<CacheEntry & { type: string, uri: string }> = data[CacheOutputGoalDataKey] || [];
 
     for (const entry of entries.filter(e => !!e.type && !!e.classifier && !!e.uri)) {

lib/core/log/RolarProgressLog.ts

@@ -23,7 +23,7 @@ import {
 import { logger } from "@atomist/automation-client/lib/util/logger";
 import { redact } from "@atomist/automation-client/lib/util/redact";
 import * as _ from "lodash";
-import os = require("os");
+import * as os from "os";
 import { format } from "../../api-helper/log/format";
 import { ProgressLog } from "../../spi/log/ProgressLog";