Move sdm-core into sdm

lib/api/goal/common/Queue.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2019 Atomist, Inc.
+ * Copyright © 2020 Atomist, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,9 +28,9 @@ import { LogSuppressor } from "../../../api-helper/log/logInterpreters";
 import {
     InProcessSdmGoalSets,
     OnAnySdmGoalSet,
+    SdmGoalFields,
     SdmGoalsByGoalSetIdAndUniqueName,
     SdmGoalState,
-    SdmGoalWithPushFields,
 } from "../../../typings/types";
 import { SoftwareDeliveryMachine } from "../../machine/SoftwareDeliveryMachine";
 import { SoftwareDeliveryMachineConfiguration } from "../../machine/SoftwareDeliveryMachineOptions";
@@ -91,6 +91,7 @@ export class Queue extends FulfillableGoal {
                     name: "InProcessSdmGoalSets",
                     variables: {
                         fetch: optsToUse.fetch + optsToUse.concurrent,
+                        offset: 0,
                         registration: [configuration.name],
                     },
                     options: QueryNoCacheOptions,
@@ -153,6 +154,7 @@ export function handleSdmGoalSetEvent(options: QueueOptions,
             name: "InProcessSdmGoalSets",
             variables: {
                 fetch: optsToUse.fetch + optsToUse.concurrent,
+                offset: 0,
                 registration: [configuration.name],
             },
             options: QueryNoCacheOptions,
@@ -169,15 +171,15 @@ export function handleSdmGoalSetEvent(options: QueueOptions,
 
 async function loadQueueGoals(goalsSets: SdmGoalSet[],
                               definition: GoalDefinition,
-                              ctx: HandlerContext): Promise<SdmGoalWithPushFields.Fragment[]> {
+                              ctx: HandlerContext): Promise<SdmGoalFields.Fragment[]> {
     return (await ctx.graphClient.query<SdmGoalsByGoalSetIdAndUniqueName.Query, SdmGoalsByGoalSetIdAndUniqueName.Variables>({
         name: "SdmGoalsByGoalSetIdAndUniqueName",
         variables: {
             goalSetId: goalsSets.map(gs => gs.goalSetId),
             uniqueName: [definition.uniqueName],
         },
         options: QueryNoCacheOptions,
-    })).SdmGoal as SdmGoalWithPushFields.Fragment[] || [];
+    })).SdmGoal as SdmGoalFields.Fragment[] || [];
 }
 
 async function startGoals(goalSets: InProcessSdmGoalSets.Query,

lib/core/goal/cache/CompressingGoalCache.ts

@@ -0,0 +1,168 @@
+/*
+ * Copyright © 2020 Atomist, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Deferred } from "@atomist/automation-client/lib/internal/util/Deferred";
+import { guid } from "@atomist/automation-client/lib/internal/util/string";
+import { GitProject } from "@atomist/automation-client/lib/project/git/GitProject";
+import * as fg from "fast-glob";
+import * as fs from "fs-extra";
+import * as JSZip from "jszip";
+import * as os from "os";
+import * as path from "path";
+import { spawnLog } from "../../../api-helper/misc/child_process";
+import { GoalInvocation } from "../../../api/goal/GoalInvocation";
+import { FileSystemGoalCacheArchiveStore } from "./FileSystemGoalCacheArchiveStore";
+import { GoalCache } from "./goalCaching";
+
+export interface GoalCacheArchiveStore {
+    /**
+     * Store a compressed goal archive
+     * @param gi The goal invocation thar triggered the caching
+     * @param classifier The classifier of the cache
+     * @param archivePath The path of the archive to be stored.
+     */
+    store(gi: GoalInvocation, classifier: string, archivePath: string): Promise<string>;
+
+    /**
+     * Remove a compressed goal archive
+     * @param gi The goal invocation thar triggered the cache removal
+     * @param classifier The classifier of the cache
+     */
+    delete(gi: GoalInvocation, classifier: string): Promise<void>;
+
+    /**
+     * Retrieve a compressed goal archive
+     * @param gi The goal invocation thar triggered the cache retrieval
+     * @param classifier The classifier of the cache
+     * @param targetArchivePath The destination path where the archive needs to be stored.
+     */
+    retrieve(gi: GoalInvocation, classifier: string, targetArchivePath: string): Promise<void>;
+}
+
+export enum CompressionMethod {
+    TAR,
+    ZIP,
+}
+
+/**
+ * Cache implementation that caches files produced by goals to an archive that can then be stored,
+ * using tar and gzip to create the archives per goal invocation (and classifier if present).
+ */
+export class CompressingGoalCache implements GoalCache {
+
+    public constructor(private readonly store: GoalCacheArchiveStore = new FileSystemGoalCacheArchiveStore(),
+                       private readonly method: CompressionMethod = CompressionMethod.TAR) {
+    }
+
+    public async put(gi: GoalInvocation,
+                     project: GitProject,
+                     files: string[],
+                     classifier?: string): Promise<string> {
+        const archiveName = "atomist-cache";
+        const teamArchiveFileName = path.join(os.tmpdir(), `${archiveName}.${guid().slice(0, 7)}`);
+        const slug = `${gi.id.owner}/${gi.id.repo}`;
+        const spawnLogOpts = {
+            log: gi.progressLog,
+            cwd: project.baseDir,
+        };
+
+        let teamArchiveFileNameWithSuffix = teamArchiveFileName;
+        if (this.method === CompressionMethod.TAR) {
+            const tarResult = await spawnLog("tar", ["-cf", teamArchiveFileName, ...files], spawnLogOpts);
+            if (tarResult.code) {
+                gi.progressLog.write(`Failed to create tar archive '${teamArchiveFileName}' for ${slug}`);
+                return undefined;
+            }
+            const gzipResult = await spawnLog("gzip", ["-3", teamArchiveFileName], spawnLogOpts);
+            if (gzipResult.code) {
+                gi.progressLog.write(`Failed to gzip tar archive '${teamArchiveFileName}' for ${slug}`);
+                return undefined;
+            }
+            teamArchiveFileNameWithSuffix += ".gz";
+        } else if (this.method === CompressionMethod.ZIP) {
+            teamArchiveFileNameWithSuffix += ".zip";
+            try {
+                const zipResult = await spawnLog("zip", ["-qr", teamArchiveFileNameWithSuffix, ...files], spawnLogOpts);
+                if (zipResult.error) {
+                    throw zipResult.error;
+                } else if (zipResult.code || zipResult.signal) {
+                    const msg = `Failed to run zip binary to create ${teamArchiveFileNameWithSuffix}: ${zipResult.code} (${zipResult.signal})`;
+                    gi.progressLog.write(msg);
+                    throw new Error(msg);
+                }
+            } catch (e) {
+                const zip = new JSZip();
+                for (const file of files) {
+                    const p = path.join(project.baseDir, file);
+                    if ((await fs.stat(p)).isFile()) {
+                        zip.file(file, fs.createReadStream(p));
+                    } else {
+                        const dirFiles = await fg(`${file}/**/*`, { cwd: project.baseDir, dot: true });
+                        for (const dirFile of dirFiles) {
+                            zip.file(dirFile, fs.createReadStream(path.join(project.baseDir, dirFile)));
+                        }
+                    }
+                }
+                const defer = new Deferred<string>();
+                zip.generateNodeStream({
+                    type: "nodebuffer",
+                    streamFiles: true,
+                    compression: "DEFLATE",
+                    compressionOptions: { level: 6 },
+                })
+                    .pipe(fs.createWriteStream(teamArchiveFileNameWithSuffix))
+                    .on("finish", () => {
+                        defer.resolve(teamArchiveFileNameWithSuffix);
+                    });
+                await defer.promise;
+            }
+        }
+        return this.store.store(gi, classifier, teamArchiveFileNameWithSuffix);
+    }
+
+    public async remove(gi: GoalInvocation, classifier?: string): Promise<void> {
+        await this.store.delete(gi, classifier);
+    }
+
+    public async retrieve(gi: GoalInvocation, project: GitProject, classifier?: string): Promise<void> {
+        const archiveName = "atomist-cache";
+        const teamArchiveFileName = path.join(os.tmpdir(), `${archiveName}.${guid().slice(0, 7)}`);
+        await this.store.retrieve(gi, classifier, teamArchiveFileName);
+        if (fs.existsSync(teamArchiveFileName)) {
+            if (this.method === CompressionMethod.TAR) {
+                await spawnLog("tar", ["-xzf", teamArchiveFileName], {
+                    log: gi.progressLog,
+                    cwd: project.baseDir,
+                });
+            } else if (this.method === CompressionMethod.ZIP) {
+                const zip = await JSZip.loadAsync(await fs.readFile(teamArchiveFileName));
+                for (const file in zip.files) {
+                    if (zip.files.hasOwnProperty(file)) {
+                        const entry = zip.file(file);
+                        if (!!entry) {
+                            const p = path.join(project.baseDir, file);
+                            await fs.ensureDir(path.dirname(p));
+                            await fs.writeFile(path.join(project.baseDir, file), await zip.file(file).async("nodebuffer"));
+                        }
+                    }
+                }
+            }
+        } else {
+            throw Error("No cache entry");
+        }
+    }
+
+}

lib/core/goal/cache/FileSystemGoalCacheArchiveStore.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright © 2020 Atomist, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from "fs-extra";
+import * as os from "os";
+import * as path from "path";
+import { spawnLog } from "../../../api-helper/misc/child_process";
+import { GoalInvocation } from "../../../api/goal/GoalInvocation";
+import { CacheConfiguration } from "../../../api/machine/SoftwareDeliveryMachineOptions";
+import { GoalCacheArchiveStore } from "./CompressingGoalCache";
+
+/**
+ * Goal archive store that stores the compressed archives into the SDM cache directory.
+ */
+export class FileSystemGoalCacheArchiveStore implements GoalCacheArchiveStore {
+    private static readonly archiveName: string = "cache.tar.gz";
+
+    public async store(gi: GoalInvocation, classifier: string, archivePath: string): Promise<string> {
+        const cacheDir = await FileSystemGoalCacheArchiveStore.getCacheDirectory(gi, classifier);
+        const archiveName = FileSystemGoalCacheArchiveStore.archiveName;
+        const archiveFileName = path.join(cacheDir, archiveName);
+        await spawnLog("mv", [archivePath, archiveFileName], {
+            log: gi.progressLog,
+        });
+        return archiveFileName;
+    }
+
+    public async delete(gi: GoalInvocation, classifier: string): Promise<void> {
+        const cacheDir = await FileSystemGoalCacheArchiveStore.getCacheDirectory(gi, classifier);
+        const archiveName = FileSystemGoalCacheArchiveStore.archiveName;
+        const archiveFileName = path.join(cacheDir, archiveName);
+        await spawnLog("rm", ["-f", archiveFileName], {
+            log: gi.progressLog,
+        });
+    }
+
+    public async retrieve(gi: GoalInvocation, classifier: string, targetArchivePath: string): Promise<void> {
+        const cacheDir = await FileSystemGoalCacheArchiveStore.getCacheDirectory(gi, classifier);
+        const archiveName = FileSystemGoalCacheArchiveStore.archiveName;
+        const archiveFileName = path.join(cacheDir, archiveName);
+        await spawnLog("cp", [archiveFileName, targetArchivePath], {
+            log: gi.progressLog,
+        });
+    }
+
+    private static async getCacheDirectory(gi: GoalInvocation, classifier: string = "default"): Promise<string> {
+        const defaultCachePath = path.join(os.homedir(), ".atomist", "cache");
+        const possibleCacheConfiguration = gi.configuration.sdm.cache as (CacheConfiguration["cache"] | undefined);
+        const sdmCacheDir = possibleCacheConfiguration ? (possibleCacheConfiguration.path || defaultCachePath) : defaultCachePath;
+        const cacheDir = path.join(sdmCacheDir, classifier);
+        await fs.mkdirs(cacheDir);
+        return cacheDir;
+    }
+}

lib/core/goal/cache/NoOpGoalCache.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright © 2020 Atomist, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GitProject } from "@atomist/automation-client/lib/project/git/GitProject";
+import { Project } from "@atomist/automation-client/lib/project/Project";
+import { logger } from "@atomist/automation-client/lib/util/logger";
+import { GoalInvocation } from "../../../api/goal/GoalInvocation";
+import { GoalCache } from "./goalCaching";
+
+/**
+ * Cache implementation that doesn't cache anything and will always trigger the fallback.
+ */
+export class NoOpGoalCache implements GoalCache {
+    public async put(gi: GoalInvocation, project: GitProject, files: string[], classifier?: string): Promise<string> {
+        logger.warn(`No-Op goal cache in use; no cache will be preserved!`);
+        return undefined;
+    }
+
+    public async remove(gi: GoalInvocation, classifier?: string): Promise<void> {
+        logger.warn(`No-Op goal cache in use; no cache will be removed!`);
+    }
+
+    public async retrieve(gi: GoalInvocation, project: Project, classifier?: string): Promise<void> {
+        throw Error("No cache entry");
+    }
+}